From c22b878e3da511542ca800111b3241f2c26c5ac5 Mon Sep 17 00:00:00 2001 From: "yuyinglu.yyl" Date: Tue, 10 Jun 2025 20:03:52 +0800 Subject: [PATCH 1/5] init commit for xset util package --- controller/expectations/cache_expectation.go | 14 + controller/merge/utils.go | 56 ++ controller/revision/revision_manager.go | 419 +++++++++ controller/revision/revision_manager_test.go | 333 +++++++ controller/utils/error.go | 40 + controller/utils/error_test.go | 54 ++ controller/utils/finalizer.go | 64 ++ controller/utils/slow_start.go | 75 ++ controller/utils/slow_start_test.go | 107 +++ go.mod | 5 +- go.sum | 5 + hack/boilerplate.go.txt | 2 +- xset/api/opslifecycle_types.go | 49 + xset/api/xset_controller_types.go | 45 + xset/api/xset_types.go | 184 ++++ xset/api/zz_generated.deepcopy.go | 199 ++++ xset/opslifecycle/const.go | 76 ++ xset/opslifecycle/default_opslifecycle.go | 73 ++ xset/opslifecycle/utils.go | 327 +++++++ xset/opslifecycle/utils_test.go | 156 ++++ xset/resourcecontexts/resource_context.go | 241 +++++ xset/revisionadapter/revision_adapter.go | 110 +++ xset/synccontrols/const.go | 46 + xset/synccontrols/inexclude.go | 64 ++ xset/synccontrols/inexclude_test.go | 300 ++++++ xset/synccontrols/sync_control.go | 911 +++++++++++++++++++ xset/synccontrols/types.go | 92 ++ xset/synccontrols/x_replace.go | 405 +++++++++ xset/synccontrols/x_scale.go | 174 ++++ xset/synccontrols/x_update.go | 691 ++++++++++++++ xset/synccontrols/x_utils.go | 185 ++++ xset/xcontrol/target_control.go | 217 +++++ xset/xset_controller.go | 244 +++++ 33 files changed, 5960 insertions(+), 3 deletions(-) create mode 100644 controller/merge/utils.go create mode 100644 controller/revision/revision_manager.go create mode 100644 controller/revision/revision_manager_test.go create mode 100644 controller/utils/error.go create mode 100644 controller/utils/error_test.go create mode 100644 controller/utils/finalizer.go create mode 100644 controller/utils/slow_start.go create mode 100644 controller/utils/slow_start_test.go create mode 100644 xset/api/opslifecycle_types.go create mode 100644 xset/api/xset_controller_types.go create mode 100644 xset/api/xset_types.go create mode 100644 xset/api/zz_generated.deepcopy.go create mode 100644 xset/opslifecycle/const.go create mode 100644 xset/opslifecycle/default_opslifecycle.go create mode 100644 xset/opslifecycle/utils.go create mode 100644 xset/opslifecycle/utils_test.go create mode 100644 xset/resourcecontexts/resource_context.go create mode 100644 xset/revisionadapter/revision_adapter.go create mode 100644 xset/synccontrols/const.go create mode 100644 xset/synccontrols/inexclude.go create mode 100644 xset/synccontrols/inexclude_test.go create mode 100644 xset/synccontrols/sync_control.go create mode 100644 xset/synccontrols/types.go create mode 100644 xset/synccontrols/x_replace.go create mode 100644 xset/synccontrols/x_scale.go create mode 100644 xset/synccontrols/x_update.go create mode 100644 xset/synccontrols/x_utils.go create mode 100644 xset/xcontrol/target_control.go create mode 100644 xset/xset_controller.go diff --git a/controller/expectations/cache_expectation.go b/controller/expectations/cache_expectation.go index 4c4f8608..8e5f5271 100644 --- a/controller/expectations/cache_expectation.go +++ b/controller/expectations/cache_expectation.go @@ -61,6 +61,10 @@ func NewxCacheExpectations(reader client.Reader, scheme *runtime.Scheme, clock c } } +func (r *CacheExpectations) CreateExpectations(controllerKey string) (*CacheExpectation, error) { + return r.initExpectations(controllerKey) +} + // GetExpectations returns the ControlleeExpectations of the given controller. func (r *CacheExpectations) GetExpectations(controllerKey string) (*CacheExpectation, bool, error) { exp, exists, err := r.GetByKey(controllerKey) @@ -182,6 +186,16 @@ func (e *CacheExpectation) Fulfilled() bool { return satisfied } +func (e *CacheExpectation) FulFilledFor(gvk schema.GroupVersionKind, namespace, name string) bool { + key := e.getKey(gvk, namespace, name) + item, ok, err := e.items.GetByKey(key) + if err != nil || !ok { + return true + } + eitem := item.(*CacheExpectationItem) + return eitem.Fulfilled() +} + func (e *CacheExpectation) ExpectCreation(gvk schema.GroupVersionKind, namespace, name string) error { return e.expect(e.getKey(gvk, namespace, name), e.creationObserved(gvk, namespace, name)) } diff --git a/controller/merge/utils.go b/controller/merge/utils.go new file mode 100644 index 00000000..78bd0721 --- /dev/null +++ b/controller/merge/utils.go @@ -0,0 +1,56 @@ +/* + * Copyright 2024-2025 KusionStack Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package merge + +import ( + "encoding/json" + + "k8s.io/apimachinery/pkg/util/strategicpatch" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// ThreeWayMergeToTarget Use three-way merge to get a updated instance. +func ThreeWayMergeToTarget(currentRevisionTarget, updateRevisionTarget, currentTarget, emptyObj client.Object) error { + currentRevisionTargetBytes, err := json.Marshal(currentRevisionTarget) + if err != nil { + return err + } + updateRevisionTargetBytes, err := json.Marshal(updateRevisionTarget) + if err != nil { + return err + } + + // 1. find the extra changes based on current revision + patch, err := strategicpatch.CreateTwoWayMergePatch(currentRevisionTargetBytes, updateRevisionTargetBytes, emptyObj) + if err != nil { + return err + } + + // 2. apply above changes to current target object + // We don't apply the diff between currentTarget and currentRevisionTarget to updateRevisionTarget, + // because the TargetTemplate changes should have the highest priority. + currentTargetBytes, err := json.Marshal(currentTarget) + if err != nil { + return err + } + if updateRevisionTargetBytes, err = strategicpatch.StrategicMergePatch(currentTargetBytes, patch, emptyObj); err != nil { + return err + } + + err = json.Unmarshal(updateRevisionTargetBytes, currentTarget) + return err +} diff --git a/controller/revision/revision_manager.go b/controller/revision/revision_manager.go new file mode 100644 index 00000000..93eb4c62 --- /dev/null +++ b/controller/revision/revision_manager.go @@ -0,0 +1,419 @@ +/* +Copyright 2017 The Kubernetes Authors. +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package revision + +import ( + "bytes" + "context" + "fmt" + "hash/fnv" + "sort" + "strconv" + + apps "k8s.io/api/apps/v1" + apiequality "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/rand" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + + "kusionstack.io/kube-utils/controller/refmanager" +) + +const ControllerRevisionHashLabel = "controller.kubernetes.io/hash" + +type OwnerAdapter interface { + GetSelector(obj metav1.Object) *metav1.LabelSelector + GetCollisionCount(obj metav1.Object) *int32 + GetHistoryLimit(obj metav1.Object) int32 + GetPatch(obj metav1.Object) ([]byte, error) + GetCurrentRevision(obj metav1.Object) string + IsInUsed(obj metav1.Object, controllerRevision string) bool +} + +func NewRevisionManager(client client.Client, scheme *runtime.Scheme, ownerGetter OwnerAdapter) *RevisionManager { + return &RevisionManager{ + Client: client, + scheme: scheme, + ownerGetter: ownerGetter, + } +} + +type RevisionManager struct { + client.Client + + scheme *runtime.Scheme + ownerGetter OwnerAdapter +} + +// controlledHistories returns all ControllerRevisions controlled by the given DaemonSet. +// This also reconciles ControllerRef by adopting/orphaning. +// Note that returned histories are pointers to objects in the cache. +// If you want to modify one, you need to deep-copy it first. +func controlledHistories(c client.Client, owner client.Object, labelSelector *metav1.LabelSelector) ([]*apps.ControllerRevision, error) { + // List all histories to include those that don't match the selector anymore + // but have a ControllerRef pointing to the controller. + selector, err := metav1.LabelSelectorAsSelector(labelSelector) + if err != nil { + return nil, err + } + histories := &apps.ControllerRevisionList{} + if err := c.List(context.TODO(), histories, &client.ListOptions{Namespace: owner.GetNamespace(), LabelSelector: selector}); err != nil { + return nil, err + } + + // Use ControllerRefManager to adopt/orphan as needed. + writer := refmanager.NewOwnerRefWriter(c) + matcher, err := refmanager.LabelSelectorAsMatch(labelSelector) + if err != nil { + return nil, fmt.Errorf("fail to create labelSelector matcher: %s", err.Error()) + } + + refManager := refmanager.NewObjectControllerRefManager(writer, owner, owner.GetObjectKind().GroupVersionKind(), matcher) + + mts := make([]metav1.Object, len(histories.Items)) + for i := range histories.Items { + mts[i] = &histories.Items[i] + } + claims, err := refManager.ClaimAllOf(context.TODO(), mts) + if err != nil { + return nil, err + } + + claimHistories := make([]*apps.ControllerRevision, len(claims)) + for i, mt := range claims { + claimHistories[i] = mt.(*apps.ControllerRevision) + } + + return claimHistories, nil +} + +// ConstructRevisions returns the current and update ControllerRevisions for set. It also +// returns a collision count that records the number of name collisions set saw when creating +// new ControllerRevisions. This count is incremented on every name collision and is used in +// building the ControllerRevision names for name collision avoidance. This method may create +// a new revision, or modify the Revision of an existing revision if an update to set is detected. +// This method expects that revisions is sorted when supplied. +func (rm *RevisionManager) ConstructRevisions(set client.Object, dryRun bool) (*apps.ControllerRevision, *apps.ControllerRevision, []*apps.ControllerRevision, *int32, bool, error) { + var currentRevision, updateRevision *apps.ControllerRevision + revisions, err := controlledHistories(rm.Client, set, rm.ownerGetter.GetSelector(set)) + if err != nil { + return currentRevision, updateRevision, nil, rm.ownerGetter.GetCollisionCount(set), false, err + } + + SortControllerRevisions(revisions) + if cleanedRevision, err := rm.cleanExpiredRevision(set, &revisions); err != nil { + return currentRevision, updateRevision, nil, rm.ownerGetter.GetCollisionCount(set), false, err + } else { + revisions = *cleanedRevision + } + + collisionCount := new(int32) + if rm.ownerGetter.GetCollisionCount(set) != nil { + collisionCount = rm.ownerGetter.GetCollisionCount(set) + } + // create a new revision from the current set + updateRevision, err = rm.newRevision(set, nextRevision(revisions), collisionCount) + if err != nil { + return nil, nil, nil, collisionCount, false, err + } + + // find any equivalent revisions + equalRevisions := FindEqualRevisions(revisions, updateRevision) + equalCount := len(equalRevisions) + revisionCount := len(revisions) + + createNewRevision := false + if equalCount > 0 && EqualRevision(revisions[revisionCount-1], equalRevisions[equalCount-1]) { + // if the equivalent revision is immediately prior the update revision has not changed + updateRevision = revisions[revisionCount-1] + } else if equalCount > 0 { + // if the equivalent revision is not immediately prior we will roll back by incrementing the + // Revision of the equivalent revision + equalRevisions[equalCount-1].Revision = updateRevision.Revision + rvc := equalRevisions[equalCount-1] + err = rm.Update(context.TODO(), rvc) + if err != nil { + return nil, nil, nil, collisionCount, false, err + } + equalRevisions[equalCount-1] = rvc + updateRevision = equalRevisions[equalCount-1] + } else { + if !dryRun { + // if there is no equivalent revision we create a new one + updateRevision, err = rm.createControllerRevision(context.TODO(), set, updateRevision, collisionCount) + if err != nil { + return nil, nil, nil, collisionCount, false, err + } + } + + revisions = append(revisions, updateRevision) + createNewRevision = true + } + + // attempt to find the revision that corresponds to the current revision + for i := range revisions { + if revisions[i].Name == rm.ownerGetter.GetCurrentRevision(set) { + currentRevision = revisions[i] + } + } + + // if the current revision is nil we initialize the history by setting it to the update revision + if currentRevision == nil { + currentRevision = updateRevision + } + + return currentRevision, updateRevision, revisions, collisionCount, createNewRevision, nil +} + +func (rm *RevisionManager) cleanExpiredRevision(set metav1.Object, sortedRevisions *[]*apps.ControllerRevision) (*[]*apps.ControllerRevision, error) { + limit := int(rm.ownerGetter.GetHistoryLimit(set)) + + // reserve 2 extra unused revisions for diagnose + exceedNum := len(*sortedRevisions) - limit - 2 + if exceedNum <= 0 { + return sortedRevisions, nil + } + + var cleanedRevisions []*apps.ControllerRevision + for _, revision := range *sortedRevisions { + if exceedNum == 0 || rm.ownerGetter.IsInUsed(set, revision.Name) { + cleanedRevisions = append(cleanedRevisions, revision) + continue + } + + if err := rm.Delete(context.TODO(), revision); err != nil { + return sortedRevisions, err + } + + exceedNum-- + } + + return &cleanedRevisions, nil +} + +func (rm *RevisionManager) createControllerRevision(ctx context.Context, parent metav1.Object, revision *apps.ControllerRevision, collisionCount *int32) (*apps.ControllerRevision, error) { + if collisionCount == nil { + return nil, fmt.Errorf("collisionCount should not be nil") + } + + // Clone the input + clone := revision.DeepCopy() + + var err error + // Continue to attempt to create the revision updating the name with a new hash on each iteration + for { + hash := hashControllerRevision(revision, collisionCount) + // Update the revisions name + clone.Name = controllerRevisionName(parent.GetName(), hash) + err = rm.Create(ctx, clone) + if errors.IsAlreadyExists(err) { + exists := &apps.ControllerRevision{} + err := rm.Get(ctx, types.NamespacedName{Namespace: clone.Namespace, Name: clone.Name}, exists) + if err != nil { + return nil, err + } + if bytes.Equal(exists.Data.Raw, clone.Data.Raw) { + return exists, nil + } + *collisionCount++ + continue + } + return clone, err + } +} + +// controllerRevisionName returns the Name for a ControllerRevision in the form prefix-hash. If the length +// of prefix is greater than 223 bytes, it is truncated to allow for a name that is no larger than 253 bytes. +func controllerRevisionName(prefix, hash string) string { + if len(prefix) > 223 { + prefix = prefix[:223] + } + + return fmt.Sprintf("%s-%s", prefix, hash) +} + +// hashControllerRevision hashes the contents of revision's Data using FNV hashing. If probe is not nil, the byte value +// of probe is added written to the hash as well. The returned hash will be a safe encoded string to avoid bad words. +func hashControllerRevision(revision *apps.ControllerRevision, probe *int32) string { + hf := fnv.New32() + if len(revision.Data.Raw) > 0 { + hf.Write(revision.Data.Raw) + } + if probe != nil { + hf.Write([]byte(strconv.FormatInt(int64(*probe), 10))) + } + return rand.SafeEncodeString(fmt.Sprint(hf.Sum32())) +} + +// newRevision creates a new ControllerRevision containing a patch that reapplies the target state of set. +// The Revision of the returned ControllerRevision is set to revision. If the returned error is nil, the returned +// ControllerRevision is valid. StatefulSet revisions are stored as patches that re-apply the current state of set +// to a new StatefulSet using a strategic merge patch to replace the saved state of the new StatefulSet. +func (rm *RevisionManager) newRevision(set metav1.Object, revision int64, collisionCount *int32) (*apps.ControllerRevision, error) { + patch, err := rm.ownerGetter.GetPatch(set) + if err != nil { + return nil, err + } + + runtimeObj, ok := set.(runtime.Object) + if !ok { + return nil, fmt.Errorf("revision owner %s/%s does not implement runtime Object interface", set.GetNamespace(), set.GetName()) + } + gvk, err := apiutil.GVKForObject(runtimeObj, rm.scheme) + if err != nil { + return nil, err + } + + revisionLabels := map[string]string{} + if selector := rm.ownerGetter.GetSelector(set); selector != nil { + for k, v := range selector.MatchLabels { + revisionLabels[k] = v + } + } + + cr := newControllerRevision(set, + gvk, + revisionLabels, + runtime.RawExtension{Raw: patch}, + revision, + collisionCount) + if err != nil { + return nil, err + } + + cr.Namespace = set.GetNamespace() + + return cr, nil +} + +// nextRevision finds the next valid revision number based on revisions. If the length of revisions +// is 0 this is 1. Otherwise, it is 1 greater than the largest revision's Revision. This method +// assumes that revisions has been sorted by Revision. +func nextRevision(revisions []*apps.ControllerRevision) int64 { + count := len(revisions) + if count <= 0 { + return 1 + } + return revisions[count-1].Revision + 1 +} + +// SortControllerRevisions sorts revisions by their Revision. +func SortControllerRevisions(revisions []*apps.ControllerRevision) { + sort.Sort(byRevision(revisions)) +} + +// byRevision implements sort.Interface to allow ControllerRevisions to be sorted by Revision. +type byRevision []*apps.ControllerRevision + +func (br byRevision) Len() int { + return len(br) +} + +func (br byRevision) Less(i, j int) bool { + return br[i].Revision < br[j].Revision +} + +func (br byRevision) Swap(i, j int) { + br[i], br[j] = br[j], br[i] +} + +// EqualRevision returns true if lhs and rhs are either both nil, or both have same labels and annotations, or bath point +// to non-nil ControllerRevisions that contain semantically equivalent data. Otherwise this method returns false. +func EqualRevision(lhs, rhs *apps.ControllerRevision) bool { + var lhsHash, rhsHash *uint32 + if lhs == nil || rhs == nil { + return lhs == rhs + } + + if hs, found := lhs.Labels[ControllerRevisionHashLabel]; found { + hash, err := strconv.ParseInt(hs, 10, 32) + if err == nil { + lhsHash = new(uint32) + *lhsHash = uint32(hash) + } + } + if hs, found := rhs.Labels[ControllerRevisionHashLabel]; found { + hash, err := strconv.ParseInt(hs, 10, 32) + if err == nil { + rhsHash = new(uint32) + *rhsHash = uint32(hash) + } + } + if lhsHash != nil && rhsHash != nil && *lhsHash != *rhsHash { + return false + } + return bytes.Equal(lhs.Data.Raw, rhs.Data.Raw) && apiequality.Semantic.DeepEqual(lhs.Data.Object, rhs.Data.Object) +} + +// FindEqualRevisions returns all ControllerRevisions in revisions that are equal to needle using EqualRevision as the +// equality test. The returned slice preserves the order of revisions. +func FindEqualRevisions(revisions []*apps.ControllerRevision, needle *apps.ControllerRevision) []*apps.ControllerRevision { + var eq []*apps.ControllerRevision + for i := range revisions { + if EqualRevision(revisions[i], needle) { + eq = append(eq, revisions[i]) + } + } + return eq +} + +// newControllerRevision returns a ControllerRevision with a ControllerRef pointing to parent and indicating that +// parent is of parentKind. The ControllerRevision has labels matching template labels, contains Data equal to data, and +// has a Revision equal to revision. The collisionCount is used when creating the name of the ControllerRevision +// so the name is likely unique. If the returned error is nil, the returned ControllerRevision is valid. If the +// returned error is not nil, the returned ControllerRevision is invalid for use. +func newControllerRevision(parent metav1.Object, + parentKind schema.GroupVersionKind, + templateLabels map[string]string, + data runtime.RawExtension, + revision int64, + collisionCount *int32, +) *apps.ControllerRevision { + labelMap := make(map[string]string) + for k, v := range templateLabels { + labelMap[k] = v + } + blockOwnerDeletion := true + isController := true + cr := &apps.ControllerRevision{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labelMap, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: parentKind.GroupVersion().String(), + Kind: parentKind.Kind, + Name: parent.GetName(), + UID: parent.GetUID(), + BlockOwnerDeletion: &blockOwnerDeletion, + Controller: &isController, + }, + }, + }, + Data: data, + Revision: revision, + } + hash := hashControllerRevision(cr, collisionCount) + cr.Name = controllerRevisionName(parent.GetName(), hash) + cr.Labels[ControllerRevisionHashLabel] = hash + return cr +} diff --git a/controller/revision/revision_manager_test.go b/controller/revision/revision_manager_test.go new file mode 100644 index 00000000..1617a8fe --- /dev/null +++ b/controller/revision/revision_manager_test.go @@ -0,0 +1,333 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package revision + +import ( + "context" + "encoding/json" + "fmt" + "testing" + "time" + + . "github.com/onsi/gomega" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +var ( + c client.Client + g *WithT + + selectedLabels = map[string]string{"test": "foo"} + selector, _ = metav1.ParseToLabelSelector("test=foo") +) + +func TestRevisionConstruction(t *testing.T) { + g = NewGomegaWithT(t) + testcase := "test-revision-construction" + schema := runtime.NewScheme() + corev1.AddToScheme(schema) + appsv1.AddToScheme(schema) + c = fake.NewClientBuilder().WithScheme(schema).Build() + g.Expect(createNamespace(c, testcase)).Should(Succeed()) + + deploy := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testcase, + Name: testcase, + }, + Spec: appsv1.DeploymentSpec{ + Selector: selector, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: selectedLabels, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + Image: "nginx:v1", + }, + }, + }, + }, + }, + } + g.Expect(c.Create(context.TODO(), deploy)).Should(Succeed()) + + adapter := &OwnerAdapterImpl{ + name: testcase, + selector: selector, + selectedLabels: selectedLabels, + collisionCount: 0, + historyLimit: 2, + currentRevision: "test", + inUsed: true, + } + + revisionManager := NewRevisionManager(c, schema, adapter) + currentRevision, updatedRevision, revisionList, collisionCount, createNewRevision, err := revisionManager.ConstructRevisions(deploy, false) + g.Expect(err).Should(Succeed()) + g.Expect(createNewRevision).Should(BeTrue()) + g.Expect(collisionCount).ShouldNot(BeNil()) + g.Expect(*collisionCount).Should(BeEquivalentTo(0)) + g.Expect(len(revisionList)).Should(BeEquivalentTo(1)) + g.Expect(len(currentRevision.Name)).ShouldNot(BeEquivalentTo(0)) + g.Expect(updatedRevision.Name).Should(BeEquivalentTo(currentRevision.Name)) + v1RevisionName := updatedRevision.Name + waitingCacheUpdate(deploy.Namespace, 1) + adapter.currentRevision = updatedRevision.Name + + // updating deploy spec should construct a new updated CcontrollerRevision + deploy.Spec.Template.Spec.Containers[0].Image = "nginx:v2" + currentRevision, updatedRevision, revisionList, collisionCount, createNewRevision, err = revisionManager.ConstructRevisions(deploy, false) + g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(createNewRevision).Should(BeTrue()) + g.Expect(collisionCount).ShouldNot(BeNil()) + g.Expect(*collisionCount).Should(BeEquivalentTo(0)) + g.Expect(len(revisionList)).Should(BeEquivalentTo(2)) + g.Expect(len(currentRevision.Name)).ShouldNot(BeEquivalentTo(0)) + g.Expect(len(currentRevision.Name)).ShouldNot(BeEquivalentTo(0)) + g.Expect(updatedRevision.Name).ShouldNot(BeEquivalentTo(currentRevision.Name)) + waitingCacheUpdate(deploy.Namespace, 2) + + // reconcile with same spec with current revision is updated to updated revision + adapter.currentRevision = updatedRevision.Name + currentRevision, updatedRevision, revisionList, collisionCount, createNewRevision, err = revisionManager.ConstructRevisions(deploy, false) + g.Expect(err).Should(Succeed()) + g.Expect(createNewRevision).Should(BeFalse()) + g.Expect(collisionCount).ShouldNot(BeNil()) + g.Expect(*collisionCount).Should(BeEquivalentTo(0)) + g.Expect(len(revisionList)).Should(BeEquivalentTo(2)) + g.Expect(len(currentRevision.Name)).ShouldNot(BeEquivalentTo(0)) + g.Expect(len(currentRevision.Name)).ShouldNot(BeEquivalentTo(0)) + g.Expect(updatedRevision.Name).Should(BeEquivalentTo(currentRevision.Name)) + waitingCacheUpdate(deploy.Namespace, 2) + + // updating deploy spec to old version should not construct a new updated CcontrollerRevision + deploy.Spec.Template.Spec.Containers[0].Image = "nginx:v1" + currentRevision, updatedRevision, revisionList, collisionCount, createNewRevision, err = revisionManager.ConstructRevisions(deploy, false) + g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(createNewRevision).Should(BeFalse()) + g.Expect(collisionCount).ShouldNot(BeNil()) + g.Expect(*collisionCount).Should(BeEquivalentTo(0)) + g.Expect(len(revisionList)).Should(BeEquivalentTo(2)) + g.Expect(len(currentRevision.Name)).ShouldNot(BeEquivalentTo(0)) + g.Expect(len(currentRevision.Name)).ShouldNot(BeEquivalentTo(0)) + g.Expect(updatedRevision.Name).ShouldNot(BeEquivalentTo(currentRevision.Name)) + g.Expect(updatedRevision.Name).Should(BeEquivalentTo(v1RevisionName)) +} + +func TestRevisionCleanUp(t *testing.T) { + g = NewGomegaWithT(t) + testcase := "test-revision-cleanup" + schema := runtime.NewScheme() + corev1.AddToScheme(schema) + appsv1.AddToScheme(schema) + c = fake.NewClientBuilder().WithScheme(schema).Build() + g.Expect(createNamespace(c, testcase)).Should(Succeed()) + + deploy := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testcase, + Name: testcase, + }, + Spec: appsv1.DeploymentSpec{ + Selector: selector, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: selectedLabels, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + Image: "nginx:v1", + }, + }, + }, + }, + }, + } + g.Expect(c.Create(context.TODO(), deploy)).Should(Succeed()) + + adapter := &OwnerAdapterImpl{ + name: testcase, + selector: selector, + selectedLabels: selectedLabels, + collisionCount: 0, + historyLimit: 0, // we at lease reserve 2 extra controller revisions + currentRevision: "test", + inUsed: true, + } + + revisionManager := NewRevisionManager(c, schema, adapter) + revisionManager.ConstructRevisions(deploy, false) + waitingCacheUpdate(deploy.Namespace, 1) + + deploy.Spec.Template.Spec.Containers[0].Image = "nginx:v2" + revisionManager.ConstructRevisions(deploy, false) + waitingCacheUpdate(deploy.Namespace, 2) + + deploy.Spec.Template.Spec.Containers[0].Image = "nginx:v3" + revisionManager.ConstructRevisions(deploy, false) + waitingCacheUpdate(deploy.Namespace, 3) + + revisionManager.ConstructRevisions(deploy, false) + waitingCacheUpdate(deploy.Namespace, 3) + + adapter.inUsed = false + revisionManager.ConstructRevisions(deploy, false) + waitingCacheUpdate(deploy.Namespace, 2) +} + +func TestRevisionCreation(t *testing.T) { + g = NewGomegaWithT(t) + testcase := "test-revision-creation" + schema := runtime.NewScheme() + corev1.AddToScheme(schema) + appsv1.AddToScheme(schema) + c = fake.NewClientBuilder().WithScheme(schema).Build() + g.Expect(createNamespace(c, testcase)).Should(Succeed()) + + deploy := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testcase, + Name: testcase, + }, + Spec: appsv1.DeploymentSpec{ + Selector: selector, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: selectedLabels, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "test", + Image: "nginx:v1", + }, + }, + }, + }, + }, + } + g.Expect(c.Create(context.TODO(), deploy)).Should(Succeed()) + + adapter := &OwnerAdapterImpl{ + name: testcase, + selector: selector, + selectedLabels: selectedLabels, + collisionCount: 0, + historyLimit: 0, // we at lease reserve 2 extra controller revisions + currentRevision: "test", + inUsed: true, + } + + revisionManager := NewRevisionManager(c, schema, adapter) + currentRevision, _, _, _, _, err := revisionManager.ConstructRevisions(deploy, false) + g.Expect(err).ShouldNot(HaveOccurred()) + waitingCacheUpdate(deploy.Namespace, 1) + + _, err = revisionManager.createControllerRevision(context.TODO(), deploy, currentRevision, nil) + g.Expect(err).Should(HaveOccurred()) + + var collisionCount int32 = 0 + // if new revision conflict with existing revision and their contents are equals, then will reuse the existing one + newRevision, err := revisionManager.newRevision(deploy, currentRevision.Revision+1, &collisionCount) + g.Expect(err).ShouldNot(HaveOccurred()) + newRevision, err = revisionManager.createControllerRevision(context.TODO(), deploy, newRevision, &collisionCount) + g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(newRevision.Name).Should(BeEquivalentTo(currentRevision.Name)) + + // change the data of existing revision + deployClone := deploy.DeepCopy() + deployClone.Spec.Template.Labels["foo"] = "foo" + currentRevision.Data.Raw, _ = revisionManager.ownerGetter.GetPatch(deployClone) + g.Expect(c.Update(context.TODO(), currentRevision)).Should(Succeed()) + // if their contents are not equals, it should regenerate a new name + newRevision, err = revisionManager.newRevision(deploy, currentRevision.Revision+1, &collisionCount) + g.Expect(err).ShouldNot(HaveOccurred()) + newRevision, err = revisionManager.createControllerRevision(context.TODO(), deploy, newRevision, &collisionCount) + g.Expect(err).ShouldNot(HaveOccurred()) + g.Expect(newRevision.Name).ShouldNot(BeEquivalentTo(currentRevision.Name)) +} + +func waitingCacheUpdate(namespace string, expectedRevisionCount int) { + g.Eventually(func() error { + revisionList := &appsv1.ControllerRevisionList{} + if err := c.List(context.TODO(), revisionList, &client.ListOptions{Namespace: namespace}); err != nil { + return err + } + + if len(revisionList.Items) != expectedRevisionCount { + return fmt.Errorf("expected %d, got %d\n", expectedRevisionCount, len(revisionList.Items)) + } + + return nil + }, 5*time.Second, 1*time.Second).Should(Succeed()) +} + +type OwnerAdapterImpl struct { + name string + selector *metav1.LabelSelector + selectedLabels map[string]string + collisionCount int32 + historyLimit int32 + currentRevision string + inUsed bool +} + +func (a OwnerAdapterImpl) GetSelector(obj metav1.Object) *metav1.LabelSelector { + return a.selector +} + +func (a OwnerAdapterImpl) GetCollisionCount(obj metav1.Object) *int32 { + return &a.collisionCount +} + +func (a OwnerAdapterImpl) GetHistoryLimit(obj metav1.Object) int32 { + return a.historyLimit +} + +func (a OwnerAdapterImpl) GetPatch(obj metav1.Object) ([]byte, error) { + // mock patch + return json.Marshal(obj) +} + +func (a OwnerAdapterImpl) GetCurrentRevision(obj metav1.Object) string { + return a.currentRevision +} + +func (a OwnerAdapterImpl) IsInUsed(obj metav1.Object, controllerRevision string) bool { + return a.inUsed +} + +func createNamespace(c client.Client, namespaceName string) error { + ns := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: namespaceName, + }, + } + + return c.Create(context.TODO(), ns) +} diff --git a/controller/utils/error.go b/controller/utils/error.go new file mode 100644 index 00000000..70a2648d --- /dev/null +++ b/controller/utils/error.go @@ -0,0 +1,40 @@ +/* +Copyright 2024 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import "fmt" + +func AggregateErrors(errs []error) error { + if len(errs) == 0 { + return nil + } else if len(errs) == 1 { + return errs[0] + } + + var aggErr error + for _, currErr := range errs { + if currErr == nil { + continue + } + if aggErr == nil { + aggErr = currErr + continue + } + aggErr = fmt.Errorf("%w; %w", aggErr, currErr) + } + return aggErr +} diff --git a/controller/utils/error_test.go b/controller/utils/error_test.go new file mode 100644 index 00000000..7ec53e96 --- /dev/null +++ b/controller/utils/error_test.go @@ -0,0 +1,54 @@ +/* +Copyright 2024 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "fmt" + "testing" +) + +func TestError(t *testing.T) { + var errs []error + err1 := fmt.Errorf("error 1") + err2 := fmt.Errorf("error 2") + var err3 error + + actual := AggregateErrors(errs) + if actual != nil { + t.Fatalf("expect %v equal to nil", actual) + } + + errs = append(errs, err1) + actual = AggregateErrors(errs) + if actual.Error() != err1.Error() { + t.Fatalf("expect %v equal to %v", actual, err1) + } + + errs = append(errs, err2) + actual = AggregateErrors(errs) + expected := fmt.Errorf("%w; %w", errs[0], errs[1]) + if actual.Error() != expected.Error() { + t.Fatalf("expect %v equal to %v", actual, expected) + } + + errs = append(errs, err3) + actual = AggregateErrors(errs) + expected = fmt.Errorf("%w; %w", errs[0], errs[1]) + if actual.Error() != expected.Error() { + t.Fatalf("expect %v equal to %v", actual, expected) + } +} diff --git a/controller/utils/finalizer.go b/controller/utils/finalizer.go new file mode 100644 index 00000000..d69c9732 --- /dev/null +++ b/controller/utils/finalizer.go @@ -0,0 +1,64 @@ +/* + Copyright 2023 The KusionStack Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package utils + +import ( + "context" + + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +func RemoveFinalizer(ctx context.Context, c client.Client, obj client.Object, finalizer string) error { + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + controllerutil.RemoveFinalizer(obj, finalizer) + var updateErr error + if updateErr = c.Update(ctx, obj); updateErr == nil { + return nil + } + if err := c.Get(ctx, types.NamespacedName{Namespace: obj.GetNamespace(), Name: obj.GetName()}, obj); err != nil { + return err + } + return updateErr + }) +} + +func AddFinalizer(ctx context.Context, c client.Client, obj client.Object, finalizer string) error { + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + controllerutil.AddFinalizer(obj, finalizer) + var updateErr error + if updateErr = c.Update(ctx, obj); updateErr == nil { + return nil + } + if err := c.Get(ctx, types.NamespacedName{Namespace: obj.GetNamespace(), Name: obj.GetName()}, obj); err != nil { + return err + } + return updateErr + }) +} + +func ContainsFinalizer(obj client.Object, finalizer string) bool { + for _, f := range obj.GetFinalizers() { + if f == finalizer { + return true + } + } + + return false +} diff --git a/controller/utils/slow_start.go b/controller/utils/slow_start.go new file mode 100644 index 00000000..5a506689 --- /dev/null +++ b/controller/utils/slow_start.go @@ -0,0 +1,75 @@ +/* +Copyright 2016 The Kubernetes Authors. +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import "sync" + +const ( + SlowStartInitialBatchSize = 1 +) + +func intMin(l, r int) int { + if l < r { + return l + } + + return r +} + +// SlowStartBatch tries to call the provided function a total of 'count' times, +// starting slow to check for errors, then speeding up if calls succeed. +// +// It groups the calls into batches, starting with a group of initialBatchSize. +// Within each batch, it may call the function multiple times concurrently. +// +// If a whole batch succeeds, the next batch may get exponentially larger. +// If there are any failures in a batch, all remaining batches are skipped +// after waiting for the current batch to complete. +// +// It returns the number of successful calls to the function. +func SlowStartBatch(count, initialBatchSize int, shortCircuit bool, fn func(int, error) error) (int, error) { + remaining := count + successes := 0 + index := 0 + var gotErr error + for batchSize := intMin(remaining, initialBatchSize); batchSize > 0; batchSize = intMin(2*batchSize, remaining) { + errCh := make(chan error, batchSize) + var wg sync.WaitGroup + wg.Add(batchSize) + for i := 0; i < batchSize; i++ { + go func(index int) { + defer wg.Done() + if err := fn(index, gotErr); err != nil { + errCh <- err + } + }(index) + index++ + } + wg.Wait() + curSuccesses := batchSize - len(errCh) + successes += curSuccesses + if len(errCh) > 0 { + gotErr = <-errCh + if shortCircuit { + return successes, gotErr + } + } + remaining -= batchSize + } + return successes, gotErr +} diff --git a/controller/utils/slow_start_test.go b/controller/utils/slow_start_test.go new file mode 100644 index 00000000..95adebc8 --- /dev/null +++ b/controller/utils/slow_start_test.go @@ -0,0 +1,107 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "fmt" + "sync" + "testing" +) + +const expectedErrMsg = "expected error" + +func TestSlowStartBatch(t *testing.T) { + testcases := []struct { + name string + count int + errorIndex *int + shortcut bool + + expectedCallCount int + expectedSuccessCount int + }{ + { + name: "happy pass", + count: 10, + errorIndex: nil, + shortcut: false, + expectedCallCount: 10, + expectedSuccessCount: 10, + }, + { + name: "failed without shortcut", + count: 10, + errorIndex: intPointer(5), + shortcut: false, + expectedCallCount: 10, + expectedSuccessCount: 9, + }, + { + name: "failed without shortcut", + count: 10, + errorIndex: intPointer(5), + shortcut: true, + expectedCallCount: 7, // 1 count in batch 1 + 2 counts in batch 2 + 4 counts in batch 3 + expectedSuccessCount: 6, + }, + } + + for _, testcase := range testcases { + callCounter := intPointer(0) + successCount, err := SlowStartBatch(testcase.count, 1, testcase.shortcut, buildFn(callCounter, testcase.errorIndex)) + if testcase.errorIndex == nil { + if err != nil { + t.Fatalf("case %s has unexpected err: %s", testcase.name, err) + } + } else { + if err == nil { + t.Fatalf("case %s has no expected err", testcase.name) + } else if err.Error() != expectedErrMsg { + t.Fatalf("case %s has expected err with unexpected message: %s", testcase.name, err) + } + } + + if successCount != testcase.expectedSuccessCount { + t.Fatalf("case %s gets unexpected success count: expected %d, got %d", testcase.name, testcase.expectedSuccessCount, successCount) + } + + if *callCounter != testcase.expectedCallCount { + t.Fatalf("case %s gets unexpected call count: expected %d, got %d", testcase.name, testcase.expectedCallCount, *callCounter) + } + } +} + +func buildFn(callCounter, errIdx *int) func(int, error) error { + lock := sync.Mutex{} + return func(i int, err error) error { + lock.Lock() + defer func() { + *callCounter++ + lock.Unlock() + }() + + if errIdx != nil && *callCounter == *errIdx { + return fmt.Errorf(expectedErrMsg) + } + + return nil + } +} + +func intPointer(val int) *int { + return &val +} diff --git a/go.mod b/go.mod index a23c2f52..e39d2717 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( k8s.io/klog/v2 v2.100.1 k8s.io/kubectl v0.28.4 k8s.io/utils v0.0.0-20240102154912-e7106e64919e - kusionstack.io/kube-api v0.2.0 + kusionstack.io/kube-api v0.6.6 sigs.k8s.io/controller-runtime v0.17.3 ) @@ -31,6 +31,7 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/docker/distribution v2.8.2+incompatible // indirect github.com/evanphx/json-patch v5.7.0+incompatible // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-logr/zapr v1.2.4 // indirect @@ -50,6 +51,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/nxadm/tail v1.4.8 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.5.0 // indirect @@ -120,6 +122,5 @@ replace ( k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.22.2 k8s.io/system-validators => k8s.io/system-validators v1.5.0 k8s.io/utils => k8s.io/utils v0.0.0-20240102154912-e7106e64919e - kusionstack.io/kube-api => kusionstack.io/kube-api v0.2.0 sigs.k8s.io/controller-runtime => sigs.k8s.io/controller-runtime v0.10.3 ) diff --git a/go.sum b/go.sum index a27b33d2..8cce733a 100644 --- a/go.sum +++ b/go.sum @@ -100,6 +100,8 @@ github.com/daviddengcn/go-colortext v0.0.0-20160507010035-511bcaf42ccd/go.mod h1 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= +github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= @@ -338,6 +340,7 @@ github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/gomega v1.15.0 h1:WjP/FQ/sk43MRmnEcT+MlDw2TFvkrXlprrPST/IudjU= github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= @@ -854,6 +857,8 @@ k8s.io/utils v0.0.0-20240102154912-e7106e64919e h1:eQ/4ljkx21sObifjzXwlPKpdGLrCf k8s.io/utils v0.0.0-20240102154912-e7106e64919e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= kusionstack.io/kube-api v0.2.0 h1:40SHCpm9RdabTUTVjhsHWoX+h7djy4jMYYTcbnJ9SQc= kusionstack.io/kube-api v0.2.0/go.mod h1:fYwuojoLs71ox8uyvyKNsJU4CtPtptSfJ3PUSt/3Hgg= +kusionstack.io/kube-api v0.6.6 h1:gMLUQL/eectQxkosnlv1m/R2xieY2crETliWRcxBICg= +kusionstack.io/kube-api v0.6.6/go.mod h1:J0+EHiroG/88X904Y9TV9iMRcoEuD5tXMTLMBDSwM+Y= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.22/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/controller-runtime v0.10.3 h1:s5Ttmw/B4AuIbwrXD3sfBkXwnPMMWrqpVj4WRt1dano= diff --git a/hack/boilerplate.go.txt b/hack/boilerplate.go.txt index e9d71f2e..d99d9a22 100644 --- a/hack/boilerplate.go.txt +++ b/hack/boilerplate.go.txt @@ -1,5 +1,5 @@ /** - * Copyright 2024 KusionStack Authors. + * Copyright 2024-2025 KusionStack Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/xset/api/opslifecycle_types.go b/xset/api/opslifecycle_types.go new file mode 100644 index 00000000..494d12c4 --- /dev/null +++ b/xset/api/opslifecycle_types.go @@ -0,0 +1,49 @@ +/* + * Copyright 2024-2025 KusionStack Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package api + +import ( + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type OperationType string + +var ( + OpsLifecycleTypeUpdate OperationType = "update" + OpsLifecycleTypeScaleIn OperationType = "scale-in" + OpsLifecycleTypeDelete OperationType = "delete" +) + +type UpdateFunc func(object client.Object) (bool, error) + +// LifecycleAdapter helps CRD Operators to easily access TargetOpsLifecycle +type LifecycleAdapter interface { + // GetID indicates ID of one TargetOpsLifecycle + GetID() string + + // GetType indicates type for an Operator + GetType() OperationType + + // AllowMultiType indicates whether multiple IDs which have the same Type are allowed + AllowMultiType() bool + + // WhenBegin will be executed when begin a lifecycle + WhenBegin(target client.Object) (needUpdate bool, err error) + + // WhenFinish will be executed when finish a lifecycle + WhenFinish(target client.Object) (needUpdate bool, err error) +} diff --git a/xset/api/xset_controller_types.go b/xset/api/xset_controller_types.go new file mode 100644 index 00000000..7c9cdca4 --- /dev/null +++ b/xset/api/xset_controller_types.go @@ -0,0 +1,45 @@ +/* + * Copyright 2024-2025 KusionStack Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package api + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type XSetController interface { + ControllerName() string + FinalizerName() string + XSetMeta() metav1.TypeMeta + XMeta() metav1.TypeMeta + + EmptyXSetObject() XSetObject + EmptyXObject() client.Object + EmptyXObjectList() client.ObjectList + + GetXSetSpec(object XSetObject) *XSetSpec + UpdateScaleStrategy(object XSetObject, scaleStrategy *ScaleStrategy) (err error) + GetXSetStatus(object XSetObject) *XSetStatus + SetXSetStatus(object XSetObject, status *XSetStatus) + + GetScaleInOpsLifecycleAdapter() LifecycleAdapter + GetUpdateOpsLifecycleAdapter() LifecycleAdapter + + GetXTemplate(object XSetObject) client.Object + CheckReady(object client.Object) bool +} + +type XSetObject client.Object diff --git a/xset/api/xset_types.go b/xset/api/xset_types.go new file mode 100644 index 00000000..0f78c27e --- /dev/null +++ b/xset/api/xset_types.go @@ -0,0 +1,184 @@ +/* + * Copyright 2024-2025 KusionStack Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package api + +// +k8s:deepcopy-gen=file + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type XSetConditionType string + +const ( + XSetScale XSetConditionType = "Scale" + XSetUpdate XSetConditionType = "Update" +) + +type XSetSpec struct { + // Indicates that the scaling and updating is paused and will not be processed by the + // XSet controller. + // +optional + Paused bool `json:"paused,omitempty"` + // Replicas is the desired number of replicas of the given Template. + // These are replicas in the sense that they are instantiations of the + // same Template, but individual replicas also have a consistent identity. + // If unspecified, defaults to 0. + // +optional + Replicas *int32 `json:"replicas,omitempty"` + + // Selector is a label query over targets that should match the replica count. + // It must match the target template's labels. + Selector *metav1.LabelSelector `json:"selector,omitempty"` + + // UpdateStrategy indicates the XSetUpdateStrategy that will be + // employed to update Targets in the XSet when a revision is made to + // Template. + // +optional + UpdateStrategy UpdateStrategy `json:"updateStrategy,omitempty"` + + // ScaleStrategy indicates the strategy detail that will be used during targets scaling. + // +optional + ScaleStrategy ScaleStrategy `json:"scaleStrategy,omitempty"` + + // Indicate the number of histories to be conserved + // If unspecified, defaults to 20 + // +optional + HistoryLimit int32 `json:"historyLimit,omitempty"` +} + +type ByPartition struct { + // Partition controls the number of targets in old revisions. + // Defaults to nil (all targets will be updated) + // +optional + Partition *int32 `json:"partition,omitempty"` +} + +type ByLabel struct{} + +// RollingUpdateStrategy is used to communicate parameter for rolling update. +type RollingUpdateStrategy struct { + // ByPartition indicates the update progress is controlled by partition value. + // +optional + ByPartition *ByPartition `json:"byPartition,omitempty"` + + // ByLabel indicates the update progress is controlled by attaching target label. + // +optional + ByLabel *ByLabel `json:"byLabel,omitempty"` +} + +type UpdateStrategy struct { + // RollingUpdate is used to communicate parameters when Type is RollingUpdateStatefulSetStrategyType. + // +optional + RollingUpdate *RollingUpdateStrategy `json:"rollingUpdate,omitempty"` + + // UpdatePolicy indicates the policy by to update targets. + // +optional + UpdatePolicy UpdateStrategyType `json:"upgradePolicy,omitempty"` + + // OperationDelaySeconds indicates how many seconds it should delay before operating update. + // +optional + OperationDelaySeconds *int32 `json:"operationDelaySeconds,omitempty"` +} + +type ScaleStrategy struct { + // Context indicates the pool from which to allocate Target instance ID. + // XSets are allowed to share the same Context. + // It is not allowed to change. + // Context defaults to be XSet's name. + // +optional + Context string `json:"context,omitempty"` + + // TargetToExclude indicates the syncContext which will be orphaned by XSet. + // +optional + TargetToExclude []string `json:"targetToExclude,omitempty"` + + // TargetToInclude indicates the syncContext which will be adapted by XSet. + // +optional + TargetToInclude []string `json:"targetToInclude,omitempty"` + + // TargetToDelete indicates the syncContext which will be deleted by XSet. + // +optional + TargetToDelete []string `json:"targetToDelete,omitempty"` + + // OperationDelaySeconds indicates how many seconds it should delay before operating scale. + // +optional + OperationDelaySeconds *int32 `json:"operationDelaySeconds,omitempty"` +} + +// UpdateStrategyType is a string enumeration type that enumerates +// all possible ways we can update a Target when updating application +type UpdateStrategyType string + +const ( + // XSetRecreateTargetUpdateStrategyType indicates that XSet will always update Target by deleting and recreate it. + XSetRecreateTargetUpdateStrategyType UpdateStrategyType = "Recreate" + // XSetInPlaceIfPossibleTargetUpdateStrategyType indicates thath XSet will try to update Target by in-place update + // when it is possible. Recently, only Target image can be updated in-place. Any other Target spec change will make the + // policy fall back to XSetRecreateTargetUpdateStrategyType. + XSetInPlaceIfPossibleTargetUpdateStrategyType UpdateStrategyType = "InPlaceIfPossible" + // XSetInPlaceOnlyTargetUpdateStrategyType indicates that XSet will always update Target in-place, instead of + // recreating target. It will encounter an error on original Kubernetes cluster. + XSetInPlaceOnlyTargetUpdateStrategyType UpdateStrategyType = "InPlaceOnly" + // XSetReplaceTargetUpdateStrategyType indicates that XSet will always update Target by replace, it will + // create a new Target and delete the old target when the new one service available. + XSetReplaceTargetUpdateStrategyType UpdateStrategyType = "Replace" +) + +type XSetStatus struct { + // ObservedGeneration is the most recent generation observed for this XSet. It corresponds to the + // XSet's generation, which is updated on mutation by the API Server. + // +optional + ObservedGeneration int64 `json:"observedGeneration,omitempty"` + + // CurrentRevision, if not empty, indicates the version of the XSet. + // +optional + CurrentRevision string `json:"currentRevision,omitempty"` + + // UpdatedRevision, if not empty, indicates the version of the XSet currently updated. + // +optional + UpdatedRevision string `json:"updatedRevision,omitempty"` + + // Count of hash collisions for the XSet. The XSet controller + // uses this field as a collision avoidance mechanism when it needs to + // create the name for the newest ControllerRevision. + // +optional + CollisionCount *int32 `json:"collisionCount,omitempty"` + + // ReadyReplicas indicates the number of the target with ready condition + // +optional + ReadyReplicas int32 `json:"readyReplicas,omitempty"` + + // Replicas is the most recently observed number of replicas. + // +optional + Replicas int32 `json:"replicas,omitempty"` + + // The number of targets in updated version. + // +optional + UpdatedReplicas int32 `json:"updatedReplicas,omitempty"` + + // OperatingReplicas indicates the number of targets during target ops lifecycle and not finish update-phase. + // +optional + OperatingReplicas int32 `json:"operatingReplicas,omitempty"` + + // UpdatedReadyReplicas indicates the number of the target with updated revision and ready condition + // +optional + UpdatedReadyReplicas int32 `json:"updatedReadyReplicas,omitempty"` + + // Represents the latest available observations of a XSet's current state. + // +optional + Conditions []metav1.Condition `json:"conditions,omitempty"` +} diff --git a/xset/api/zz_generated.deepcopy.go b/xset/api/zz_generated.deepcopy.go new file mode 100644 index 00000000..46192a57 --- /dev/null +++ b/xset/api/zz_generated.deepcopy.go @@ -0,0 +1,199 @@ +//go:build !ignore_autogenerated + +/* + * Copyright 2024-2025 KusionStack Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Code generated by controller-gen. DO NOT EDIT. + +package api + +import ( + "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ByLabel) DeepCopyInto(out *ByLabel) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ByLabel. +func (in *ByLabel) DeepCopy() *ByLabel { + if in == nil { + return nil + } + out := new(ByLabel) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ByPartition) DeepCopyInto(out *ByPartition) { + *out = *in + if in.Partition != nil { + in, out := &in.Partition, &out.Partition + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ByPartition. +func (in *ByPartition) DeepCopy() *ByPartition { + if in == nil { + return nil + } + out := new(ByPartition) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *RollingUpdateStrategy) DeepCopyInto(out *RollingUpdateStrategy) { + *out = *in + if in.ByPartition != nil { + in, out := &in.ByPartition, &out.ByPartition + *out = new(ByPartition) + (*in).DeepCopyInto(*out) + } + if in.ByLabel != nil { + in, out := &in.ByLabel, &out.ByLabel + *out = new(ByLabel) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RollingUpdateStrategy. +func (in *RollingUpdateStrategy) DeepCopy() *RollingUpdateStrategy { + if in == nil { + return nil + } + out := new(RollingUpdateStrategy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ScaleStrategy) DeepCopyInto(out *ScaleStrategy) { + *out = *in + if in.TargetToExclude != nil { + in, out := &in.TargetToExclude, &out.TargetToExclude + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.TargetToInclude != nil { + in, out := &in.TargetToInclude, &out.TargetToInclude + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.TargetToDelete != nil { + in, out := &in.TargetToDelete, &out.TargetToDelete + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.OperationDelaySeconds != nil { + in, out := &in.OperationDelaySeconds, &out.OperationDelaySeconds + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScaleStrategy. +func (in *ScaleStrategy) DeepCopy() *ScaleStrategy { + if in == nil { + return nil + } + out := new(ScaleStrategy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *UpdateStrategy) DeepCopyInto(out *UpdateStrategy) { + *out = *in + if in.RollingUpdate != nil { + in, out := &in.RollingUpdate, &out.RollingUpdate + *out = new(RollingUpdateStrategy) + (*in).DeepCopyInto(*out) + } + if in.OperationDelaySeconds != nil { + in, out := &in.OperationDelaySeconds, &out.OperationDelaySeconds + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UpdateStrategy. +func (in *UpdateStrategy) DeepCopy() *UpdateStrategy { + if in == nil { + return nil + } + out := new(UpdateStrategy) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *XSetSpec) DeepCopyInto(out *XSetSpec) { + *out = *in + if in.Replicas != nil { + in, out := &in.Replicas, &out.Replicas + *out = new(int32) + **out = **in + } + if in.Selector != nil { + in, out := &in.Selector, &out.Selector + *out = new(v1.LabelSelector) + (*in).DeepCopyInto(*out) + } + in.UpdateStrategy.DeepCopyInto(&out.UpdateStrategy) + in.ScaleStrategy.DeepCopyInto(&out.ScaleStrategy) +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new XSetSpec. +func (in *XSetSpec) DeepCopy() *XSetSpec { + if in == nil { + return nil + } + out := new(XSetSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *XSetStatus) DeepCopyInto(out *XSetStatus) { + *out = *in + if in.CollisionCount != nil { + in, out := &in.CollisionCount, &out.CollisionCount + *out = new(int32) + **out = **in + } + if in.Conditions != nil { + in, out := &in.Conditions, &out.Conditions + *out = make([]v1.Condition, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new XSetStatus. +func (in *XSetStatus) DeepCopy() *XSetStatus { + if in == nil { + return nil + } + out := new(XSetStatus) + in.DeepCopyInto(out) + return out +} diff --git a/xset/opslifecycle/const.go b/xset/opslifecycle/const.go new file mode 100644 index 00000000..4edd9614 --- /dev/null +++ b/xset/opslifecycle/const.go @@ -0,0 +1,76 @@ +/* + * Copyright 2024-2025 KusionStack Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opslifecycle + +// TargetOpsLifecycle labels +const ( + // TargetOperatingLabelPrefix indicates a Target is operating + TargetOperatingLabelPrefix = "operating.opslifecycle.kusionstack.io" + // TargetOperationTypeLabelPrefix indicates the type of operation + TargetOperationTypeLabelPrefix = "operation-type.opslifecycle.kusionstack.io" + // TargetOperationPermissionLabelPrefix indicates the permission of operation + TargetOperationPermissionLabelPrefix = "operation-permission.opslifecycle.kusionstack.io" + // TargetUndoOperationTypeLabelPrefix indicates the type of operation has been canceled + TargetUndoOperationTypeLabelPrefix = "undo-operation-type.opslifecycle.kusionstack.io" + // TargetDoneOperationTypeLabelPrefix indicates the type of operation has been done + TargetDoneOperationTypeLabelPrefix = "done-operation-type.opslifecycle.kusionstack.io" + + // TargetPreCheckLabelPrefix indicates a Target is in pre-check phase + TargetPreCheckLabelPrefix = "pre-check.opslifecycle.kusionstack.io" + // TargetPreCheckedLabelPrefix indicates a Target has finished pre-check phase + TargetPreCheckedLabelPrefix = "pre-checked.opslifecycle.kusionstack.io" + // TargetPreparingLabelPrefix indicates a Target is preparing for operation + TargetPreparingLabelPrefix = "preparing.opslifecycle.kusionstack.io" + // TargetOperateLabelPrefix indicates a Target is in operate phase + TargetOperateLabelPrefix = "operate.opslifecycle.kusionstack.io" + // TargetOperatedLabelPrefix indicates a Target has finished operate phase + TargetOperatedLabelPrefix = "operated.opslifecycle.kusionstack.io" + // TargetPostCheckLabelPrefix indicates a Target is in post-check phase + TargetPostCheckLabelPrefix = "post-check.opslifecycle.kusionstack.io" + // TargetPostCheckedLabelPrefix indicates a Target has finished post-check phase + TargetPostCheckedLabelPrefix = "post-checked.opslifecycle.kusionstack.io" + // TargetCompletingLabelPrefix indicates a Target is completing operation + TargetCompletingLabelPrefix = "completing.opslifecycle.kusionstack.io" + + // TargetServiceAvailableLabel indicates a Target is available to serve + TargetServiceAvailableLabel = "opslifecycle.kusionstack.io/service-available" + TargetPreCheckLabel = "opslifecycle.kusionstack.io/pre-checking" + TargetPreparingLabel = "opslifecycle.kusionstack.io/preparing" + TargetOperatingLabel = "opslifecycle.kusionstack.io/operating" + TargetPostCheckLabel = "opslifecycle.kusionstack.io/post-checking" + TargetCompletingLabel = "opslifecycle.kusionstack.io/completing" + TargetCreatingLabel = "opslifecycle.kusionstack.io/creating" + + // TargetStayOfflineLabel indicates a Target is not ready and available to serve + TargetStayOfflineLabel = "opslifecycle.kusionstack.io/stay-offline" + TargetPreparingDeleteLabel = "opslifecycle.kusionstack.io/preparing-to-delete" +) + +var WellKnownLabelPrefixesWithID = []string{ + TargetOperatingLabelPrefix, + TargetOperationTypeLabelPrefix, + TargetPreCheckLabelPrefix, + TargetPreCheckedLabelPrefix, + TargetPreparingLabelPrefix, + TargetDoneOperationTypeLabelPrefix, + TargetUndoOperationTypeLabelPrefix, + TargetOperateLabelPrefix, + TargetOperatedLabelPrefix, + TargetPostCheckLabelPrefix, + TargetPostCheckedLabelPrefix, + TargetCompletingLabelPrefix, +} diff --git a/xset/opslifecycle/default_opslifecycle.go b/xset/opslifecycle/default_opslifecycle.go new file mode 100644 index 00000000..804defa3 --- /dev/null +++ b/xset/opslifecycle/default_opslifecycle.go @@ -0,0 +1,73 @@ +/* + * Copyright 2024-2025 KusionStack Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opslifecycle + +import ( + "sigs.k8s.io/controller-runtime/pkg/client" + + "kusionstack.io/kube-utils/xset/api" +) + +var _ api.LifecycleAdapter = &DefaultUpdateLifecycleAdapter{} + +type DefaultUpdateLifecycleAdapter struct{} + +func (d *DefaultUpdateLifecycleAdapter) GetID() string { + return "xset" +} + +func (d *DefaultUpdateLifecycleAdapter) GetType() api.OperationType { + return api.OpsLifecycleTypeUpdate +} + +func (d *DefaultUpdateLifecycleAdapter) AllowMultiType() bool { + return true +} + +func (d *DefaultUpdateLifecycleAdapter) WhenBegin(target client.Object) (bool, error) { + setOperate(d, target) + return true, nil +} + +func (d *DefaultUpdateLifecycleAdapter) WhenFinish(target client.Object) (bool, error) { + return false, nil +} + +var _ api.LifecycleAdapter = &DefaultScaleInLifecycleAdapter{} + +type DefaultScaleInLifecycleAdapter struct{} + +func (d *DefaultScaleInLifecycleAdapter) GetID() string { + return "xset" +} + +func (d *DefaultScaleInLifecycleAdapter) GetType() api.OperationType { + return api.OpsLifecycleTypeScaleIn +} + +func (d *DefaultScaleInLifecycleAdapter) AllowMultiType() bool { + return true +} + +func (d *DefaultScaleInLifecycleAdapter) WhenBegin(target client.Object) (bool, error) { + setOperate(d, target) + return true, nil +} + +func (d *DefaultScaleInLifecycleAdapter) WhenFinish(target client.Object) (bool, error) { + return false, nil +} diff --git a/xset/opslifecycle/utils.go b/xset/opslifecycle/utils.go new file mode 100644 index 00000000..71323261 --- /dev/null +++ b/xset/opslifecycle/utils.go @@ -0,0 +1,327 @@ +/* + Copyright 2023-2025 The KusionStack Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package opslifecycle + +import ( + "context" + "fmt" + "strconv" + "strings" + "time" + + "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/controller-runtime/pkg/client" + + "kusionstack.io/kube-utils/xset/api" +) + +// IDToLabelsMap returns a map of pod id to labels map and a map of operation type to number of pods. +func IDToLabelsMap(target client.Object) (map[string]map[string]string, map[string]int, error) { + idToLabelsMap := map[string]map[string]string{} + typeToNumsMap := map[string]int{} + + ids := sets.String{} + labels := target.GetLabels() + for k := range labels { + if strings.HasPrefix(k, TargetOperatingLabelPrefix) || + strings.HasPrefix(k, TargetOperateLabelPrefix) || + strings.HasPrefix(k, TargetOperatedLabelPrefix) { + s := strings.Split(k, "/") + if len(s) < 2 { + return nil, nil, fmt.Errorf("invalid label %s", k) + } + ids.Insert(s[1]) + } + } + + for id := range ids { + for _, val := range []string{TargetOperationTypeLabelPrefix, TargetDoneOperationTypeLabelPrefix} { + operationType, ok := labels[fmt.Sprintf("%s/%s", val, id)] + if !ok { + continue + } + + if _, ok := typeToNumsMap[operationType]; !ok { + typeToNumsMap[operationType] = 1 + } else { + typeToNumsMap[operationType] += 1 + } + break + } + + for _, prefix := range WellKnownLabelPrefixesWithID { + label := fmt.Sprintf("%s/%s", prefix, id) + value, ok := labels[label] + if !ok { + continue + } + + labelsMap, ok := idToLabelsMap[id] + if !ok { + labelsMap = make(map[string]string) + idToLabelsMap[id] = labelsMap + } + labelsMap[prefix] = value + } + } + return idToLabelsMap, typeToNumsMap, nil +} + +// NumOfLifecycleOnTarget returns the nums of lifecycles on pod +func NumOfLifecycleOnTarget(target client.Object) (int, error) { + if target == nil { + return 0, nil + } + newIDToLabelsMap, _, err := IDToLabelsMap(target) + return len(newIDToLabelsMap), err +} + +var WhenBeginDelete api.UpdateFunc = func(obj client.Object) (bool, error) { + return AddLabel(obj, TargetPreparingDeleteLabel, strconv.FormatInt(time.Now().UnixNano(), 10)), nil +} + +func AddLabel(po client.Object, k, v string) bool { + labels := po.GetLabels() + if labels == nil { + labels = map[string]string{} + po.SetLabels(labels) + } + if _, ok := labels[k]; !ok { + labels[k] = v + return true + } + return false +} + +// IsDuringOps decides whether the Target is during ops or not +// DuringOps means the Target's OpsLifecycle phase is in or after PreCheck phase and before Finish phase. +func IsDuringOps(adapter api.LifecycleAdapter, obj client.Object) bool { + _, hasID := checkOperatingID(adapter, obj) + _, hasType := checkOperationType(adapter, obj) + + return hasID && hasType +} + +// Begin is used for an CRD Operator to begin a lifecycle +func Begin(c client.Client, adapter api.LifecycleAdapter, obj client.Object, updateFunc ...api.UpdateFunc) (updated bool, err error) { + if obj.GetLabels() == nil { + obj.SetLabels(map[string]string{}) + } + + operatingID, hasID := checkOperatingID(adapter, obj) + operationType, hasType := checkOperationType(adapter, obj) + var needUpdate bool + + // ensure operatingID and operationType + if hasID && hasType { + if operationType != adapter.GetType() { + err = fmt.Errorf("operatingID %s already has operationType %s", operatingID, operationType) + return false, err + } + } else { + // check another id/type = this.type + currentTypeIDs := queryByOperationType(adapter, obj) + if currentTypeIDs != nil && currentTypeIDs.Len() > 0 && !adapter.AllowMultiType() { + err = fmt.Errorf("operationType %s exists: %v", adapter.GetType(), currentTypeIDs) + return updated, err + } + + if !hasID { + needUpdate = true + setOperatingID(adapter, obj) + } + if !hasType { + needUpdate = true + setOperationType(adapter, obj) + } + } + + updated, err = DefaultUpdateAll(obj, append(updateFunc, adapter.WhenBegin)...) + if err != nil { + return updated, err + } + + if needUpdate || updated { + err = c.Update(context.Background(), obj) + return err == nil, err + } + + return false, nil +} + +// BeginWithCleaningOld is used for an CRD Operator to begin a lifecycle with cleaning the old lifecycle +func BeginWithCleaningOld(c client.Client, adapter api.LifecycleAdapter, obj client.Object, updateFunc ...api.UpdateFunc) (updated bool, err error) { + if targetInUpdateLifecycle, err := IsLifecycleOnTarget(adapter.GetID(), obj); err != nil { + return false, fmt.Errorf("fail to check %s TargetOpsLifecycle on Target %s/%s: %w", adapter.GetID(), obj.GetNamespace(), obj.GetName(), err) + } else if targetInUpdateLifecycle { + if err := Undo(c, adapter, obj); err != nil { + return false, err + } + } + return Begin(c, adapter, obj, updateFunc...) +} + +// AllowOps is used to check whether the TargetOpsLifecycle phase is in UPGRADE to do following operations. +func AllowOps(adapter api.LifecycleAdapter, operationDelaySeconds int32, obj client.Object) (requeueAfter *time.Duration, allow bool) { + if !IsDuringOps(adapter, obj) { + return nil, false + } + + startedTimestampStr, started := checkOperate(adapter, obj) + if !started || operationDelaySeconds <= 0 { + return nil, started + } + + startedTimestamp, err := strconv.ParseInt(startedTimestampStr, 10, 64) + if err != nil { + return nil, started + } + + startedTime := time.Unix(0, startedTimestamp) + duration := time.Since(startedTime) + delay := time.Duration(operationDelaySeconds) * time.Second + if duration < delay { + du := delay - duration + return &du, started + } + + return nil, started +} + +// Finish is used for an CRD Operator to finish a lifecycle +func Finish(c client.Client, adapter api.LifecycleAdapter, obj client.Object, updateFuncs ...api.UpdateFunc) (updated bool, err error) { + operatingID, hasID := checkOperatingID(adapter, obj) + operationType, hasType := checkOperationType(adapter, obj) + + if hasType && operationType != adapter.GetType() { + return false, fmt.Errorf("operatingID %s has invalid operationType %s", operatingID, operationType) + } + + var needUpdate bool + if hasID || hasType { + needUpdate = true + deleteOperatingID(adapter, obj) + } + + updated, err = DefaultUpdateAll(obj, append(updateFuncs, adapter.WhenFinish)...) + if err != nil { + return + } + if needUpdate || updated { + err = c.Update(context.Background(), obj) + return err == nil, err + } + + return false, err +} + +// Undo is used for an CRD Operator to undo a lifecycle +func Undo(c client.Client, adapter api.LifecycleAdapter, obj client.Object) error { + setUndo(adapter, obj) + return c.Update(context.Background(), obj) +} + +func checkOperatingID(adapter api.LifecycleAdapter, obj client.Object) (val string, ok bool) { + labelID := fmt.Sprintf("%s/%s", TargetOperatingLabelPrefix, adapter.GetID()) + _, ok = obj.GetLabels()[labelID] + return adapter.GetID(), ok +} + +func checkOperationType(adapter api.LifecycleAdapter, obj client.Object) (val api.OperationType, ok bool) { + labelType := fmt.Sprintf("%s/%s", TargetOperationTypeLabelPrefix, adapter.GetID()) + labelVal := obj.GetLabels()[labelType] + val = api.OperationType(labelVal) + return val, val == adapter.GetType() +} + +func checkOperate(adapter api.LifecycleAdapter, obj client.Object) (val string, ok bool) { + labelOperate := fmt.Sprintf("%s/%s", TargetOperateLabelPrefix, adapter.GetID()) + val, ok = obj.GetLabels()[labelOperate] + return +} + +func setOperatingID(adapter api.LifecycleAdapter, obj client.Object) { + labelID := fmt.Sprintf("%s/%s", TargetOperatingLabelPrefix, adapter.GetID()) + obj.GetLabels()[labelID] = fmt.Sprintf("%d", time.Now().UnixNano()) + return +} + +func setOperationType(adapter api.LifecycleAdapter, obj client.Object) { + labelType := fmt.Sprintf("%s/%s", TargetOperationTypeLabelPrefix, adapter.GetID()) + obj.GetLabels()[labelType] = string(adapter.GetType()) + return +} + +// setOperate only for test +func setOperate(adapter api.LifecycleAdapter, obj client.Object) { + labelOperate := fmt.Sprintf("%s/%s", TargetOperateLabelPrefix, adapter.GetID()) + now := time.Now().UnixNano() + obj.GetLabels()[labelOperate] = strconv.FormatInt(now, 10) + return +} + +func setUndo(adapter api.LifecycleAdapter, obj client.Object) { + labelUndo := fmt.Sprintf("%s/%s", TargetUndoOperationTypeLabelPrefix, adapter.GetID()) + obj.GetLabels()[labelUndo] = string(adapter.GetType()) +} + +func deleteOperatingID(adapter api.LifecycleAdapter, obj client.Object) { + labelID := fmt.Sprintf("%s/%s", TargetOperatingLabelPrefix, adapter.GetID()) + delete(obj.GetLabels(), labelID) + return +} + +func queryByOperationType(adapter api.LifecycleAdapter, obj client.Object) sets.String { + res := sets.String{} + valType := adapter.GetType() + + for k, v := range obj.GetLabels() { + if strings.HasPrefix(k, TargetOperationTypeLabelPrefix) && v == string(valType) { + res.Insert(k) + } + } + + return res +} + +func DefaultUpdateAll(target client.Object, updateFuncs ...api.UpdateFunc) (updated bool, err error) { + for _, updateFunc := range updateFuncs { + ok, updateErr := updateFunc(target) + if updateErr != nil { + return updated, updateErr + } + updated = updated || ok + } + return updated, nil +} + +func IsLifecycleOnTarget(operatingID string, target client.Object) (bool, error) { + if target == nil { + return false, fmt.Errorf("nil Target") + } + + labels := target.GetLabels() + if labels == nil { + return false, nil + } + + if val, ok := labels[fmt.Sprintf("%s/%s", TargetOperatingLabelPrefix, operatingID)]; ok { + return val != "", nil + } + + return false, nil +} diff --git a/xset/opslifecycle/utils_test.go b/xset/opslifecycle/utils_test.go new file mode 100644 index 00000000..70fc3b8a --- /dev/null +++ b/xset/opslifecycle/utils_test.go @@ -0,0 +1,156 @@ +/* + Copyright 2023-2025 The KusionStack Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package opslifecycle + +import ( + "context" + "fmt" + "reflect" + "testing" + + "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/sets" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "kusionstack.io/kube-utils/xset/api" +) + +const ( + mockLabelKey = "mockLabel" + mockLabelValue = "mockLabelValue" + testNamespace = "default" + testName = "target-1" +) + +var ( + allowTypes = false + scheme = runtime.NewScheme() +) + +func init() { + corev1.AddToScheme(scheme) +} + +func TestLifecycle(t *testing.T) { + c := fake.NewClientBuilder().WithScheme(scheme).Build() + g := gomega.NewGomegaWithT(t) + + a := &mockAdapter{id: "id-1", operationType: "type-1"} + b := &mockAdapter{id: "id-2", operationType: "type-1"} + + inputs := []struct { + hasOperating, hasConflictID bool + started bool + err error + allow bool + }{ + { + hasOperating: false, + started: false, + }, + { + hasOperating: true, + started: false, + }, + { + hasConflictID: true, + started: false, + err: fmt.Errorf("operationType %s exists: %v", a.GetType(), sets.NewString(fmt.Sprintf("%s/%s", TargetOperationTypeLabelPrefix, b.GetID()))), + }, + { + hasConflictID: true, + started: false, + allow: true, + err: nil, + }, + } + + for i, input := range inputs { + allowTypes = input.allow + + target := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: testNamespace, + Name: fmt.Sprintf("%s-%d", testName, i), + Labels: map[string]string{}, + Annotations: map[string]string{}, + }, + } + g.Expect(c.Create(context.TODO(), target)).Should(gomega.Succeed()) + + if input.hasOperating { + setOperatingID(a, target) + setOperationType(a, target) + a.WhenBegin(target) + } + + if input.hasConflictID { + setOperatingID(b, target) + setOperationType(b, target) + } + + _, err := Begin(c, a, target) + g.Expect(reflect.DeepEqual(err, input.err)).Should(gomega.BeTrue()) + if err != nil { + continue + } + g.Expect(target.Labels[mockLabelKey]).Should(gomega.BeEquivalentTo(mockLabelValue)) + + setOperate(a, target) + started, err := Begin(c, a, target) + g.Expect(err).ShouldNot(gomega.HaveOccurred()) + g.Expect(started).Should(gomega.BeTrue()) + g.Expect(IsDuringOps(a, target)).Should(gomega.BeTrue()) + + finished, err := Finish(c, a, target) + g.Expect(err).ShouldNot(gomega.HaveOccurred()) + g.Expect(finished).Should(gomega.BeTrue()) + g.Expect(target.Labels[mockLabelKey]).Should(gomega.BeEquivalentTo("")) + g.Expect(IsDuringOps(a, target)).Should(gomega.BeFalse()) + } +} + +type mockAdapter struct { + id string + operationType api.OperationType +} + +func (m *mockAdapter) GetID() string { + return m.id +} + +func (m *mockAdapter) GetType() api.OperationType { + return m.operationType +} + +func (m *mockAdapter) AllowMultiType() bool { + return allowTypes +} + +func (m *mockAdapter) WhenBegin(target client.Object) (bool, error) { + target.GetLabels()[mockLabelKey] = mockLabelValue + return true, nil +} + +func (m *mockAdapter) WhenFinish(target client.Object) (bool, error) { + delete(target.GetLabels(), mockLabelKey) + return true, nil +} diff --git a/xset/resourcecontexts/resource_context.go b/xset/resourcecontexts/resource_context.go new file mode 100644 index 00000000..d234f14b --- /dev/null +++ b/xset/resourcecontexts/resource_context.go @@ -0,0 +1,241 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resourcecontexts + +import ( + "context" + "fmt" + + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "kusionstack.io/kube-utils/controller/expectations" + "kusionstack.io/kube-utils/xset/api" +) + +var resourceContextGVK = appsv1alpha1.SchemeGroupVersion.WithKind("ResourceContext") + +const ( + OwnerContextKey = "Owner" + RevisionContextDataKey = "Revision" + TargetDecorationRevisionKey = "TargetDecorationRevisions" + JustCreateContextDataKey = "TargetJustCreate" + RecreateUpdateContextDataKey = "TargetRecreateUpdate" + ScaleInContextDataKey = "ScaleIn" +) + +func AllocateID(xsetControl api.XSetController, c client.Client, e *expectations.CacheExpectation, + instance api.XSetObject, defaultRevision string, replicas int, +) (map[int]*appsv1alpha1.ContextDetail, error) { + contextName := getContextName(xsetControl, instance) + targetContext := &appsv1alpha1.ResourceContext{} + notFound := false + if err := c.Get(context.TODO(), types.NamespacedName{Namespace: instance.GetNamespace(), Name: contextName}, targetContext); err != nil { + if !errors.IsNotFound(err) { + return nil, fmt.Errorf("fail to find ResourceContext %s/%s for owner %s: %s", instance.GetNamespace(), contextName, instance.GetName(), err.Error()) + } + + notFound = true + targetContext.Namespace = instance.GetNamespace() + targetContext.Name = contextName + } + + xspec := xsetControl.GetXSetSpec(instance) + // store all the IDs crossing Multiple workload + existingIDs := map[int]*appsv1alpha1.ContextDetail{} + // only store the IDs belonging to this owner + ownedIDs := map[int]*appsv1alpha1.ContextDetail{} + for i := range targetContext.Spec.Contexts { + detail := &targetContext.Spec.Contexts[i] + if detail.Contains(OwnerContextKey, instance.GetName()) { + ownedIDs[detail.ID] = detail + existingIDs[detail.ID] = detail + } else if xspec.ScaleStrategy.Context != "" { + // add other collaset targetContexts only if context pool enabled + existingIDs[detail.ID] = detail + } + } + + // if owner has enough ID, return + if len(ownedIDs) >= replicas { + return ownedIDs, nil + } + + // find new IDs for owner + candidateID := 0 + for len(ownedIDs) < replicas { + // find one new ID + for { + if _, exist := existingIDs[candidateID]; exist { + candidateID++ + continue + } + + break + } + + detail := &appsv1alpha1.ContextDetail{ + ID: candidateID, + // TODO choose just create targets' revision according to scaleStrategy + Data: map[string]string{ + OwnerContextKey: instance.GetName(), + RevisionContextDataKey: defaultRevision, + JustCreateContextDataKey: "true", + }, + } + existingIDs[candidateID] = detail + ownedIDs[candidateID] = detail + } + + if notFound { + return ownedIDs, doCreateTargetContext(xsetControl, c, instance, ownedIDs) + } + + return ownedIDs, doUpdateTargetContext(xsetControl, c, e, instance, ownedIDs, targetContext) +} + +func UpdateToTargetContext(xsetController api.XSetController, c client.Client, e *expectations.CacheExpectation, + instance api.XSetObject, ownedIDs map[int]*appsv1alpha1.ContextDetail, +) error { + contextName := getContextName(xsetController, instance) + targetContext := &appsv1alpha1.ResourceContext{} + if err := c.Get(context.TODO(), types.NamespacedName{Namespace: instance.GetNamespace(), Name: contextName}, targetContext); err != nil { + if !errors.IsNotFound(err) { + return fmt.Errorf("fail to find ResourceContext %s/%s: %w", instance.GetNamespace(), contextName, err) + } + + if len(ownedIDs) == 0 { + return nil + } + + if err := doCreateTargetContext(xsetController, c, instance, ownedIDs); err != nil { + return fmt.Errorf("fail to create ResourceContext %s/%s after not found: %w", instance.GetNamespace(), contextName, err) + } + } + + return doUpdateTargetContext(xsetController, c, e, instance, ownedIDs, targetContext) +} + +func ExtractAvailableContexts(diff int, ownedIDs map[int]*appsv1alpha1.ContextDetail, targetInstanceIDSet sets.Int) []*appsv1alpha1.ContextDetail { + var availableContexts []*appsv1alpha1.ContextDetail + if diff <= 0 { + return availableContexts + } + + idx := 0 + for id := range ownedIDs { + if _, inUsed := targetInstanceIDSet[id]; inUsed { + continue + } + + availableContexts = append(availableContexts, ownedIDs[id]) + idx++ + if idx == diff { + break + } + } + + return availableContexts +} + +func doCreateTargetContext(xsetController api.XSetController, c client.Client, instance api.XSetObject, ownerIDs map[int]*appsv1alpha1.ContextDetail) error { + contextName := getContextName(xsetController, instance) + targetContext := &appsv1alpha1.ResourceContext{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: instance.GetNamespace(), + Name: contextName, + }, + Spec: appsv1alpha1.ResourceContextSpec{ + Contexts: make([]appsv1alpha1.ContextDetail, len(ownerIDs)), + }, + } + + setContextData(ownerIDs, targetContext) + return c.Create(context.TODO(), targetContext) +} + +func doUpdateTargetContext(xsetController api.XSetController, c client.Client, e *expectations.CacheExpectation, + instance client.Object, ownedIDs map[int]*appsv1alpha1.ContextDetail, targetContext *appsv1alpha1.ResourceContext, +) error { + // store all IDs crossing all workload + existingIDs := map[int]*appsv1alpha1.ContextDetail{} + + // add other collaset targetContexts only if context pool enabled + spec := xsetController.GetXSetSpec(instance) + if spec.ScaleStrategy.Context != "" { + for i := range targetContext.Spec.Contexts { + detail := targetContext.Spec.Contexts[i] + if detail.Contains(OwnerContextKey, instance.GetName()) { + continue + } + existingIDs[detail.ID] = &detail + } + } + + for _, contextDetail := range ownedIDs { + existingIDs[contextDetail.ID] = contextDetail + } + + // delete TargetContext if it is empty + if len(existingIDs) == 0 { + err := c.Delete(context.TODO(), targetContext) + if err != nil { + if err := e.ExpectDeletion(resourceContextGVK, targetContext.Namespace, targetContext.Name); err != nil { + return err + } + } + + return err + } + + setContextData(existingIDs, targetContext) + err := c.Update(context.TODO(), targetContext) + if err != nil { + if err := e.ExpectUpdation(resourceContextGVK, targetContext.Namespace, targetContext.Name, targetContext.ResourceVersion); err != nil { + return err + } + } + + return err +} + +func getContextName(xsetControl api.XSetController, instance api.XSetObject) string { + spec := xsetControl.GetXSetSpec(instance) + if spec.ScaleStrategy.Context != "" { + return spec.ScaleStrategy.Context + } + + return instance.GetName() +} + +func setContextData(detailIDs map[int]*appsv1alpha1.ContextDetail, targetContext *appsv1alpha1.ResourceContext) { + length := len(detailIDs) + targetContext.Spec.Contexts = make([]appsv1alpha1.ContextDetail, 0, length) + idx := 0 + for len(targetContext.Spec.Contexts) < length { + for _, ok := detailIDs[idx]; !ok; { + idx += 1 + _, ok = detailIDs[idx] + } + targetContext.Spec.Contexts = append(targetContext.Spec.Contexts, *detailIDs[idx]) + idx += 1 + } +} diff --git a/xset/revisionadapter/revision_adapter.go b/xset/revisionadapter/revision_adapter.go new file mode 100644 index 00000000..ec84afd3 --- /dev/null +++ b/xset/revisionadapter/revision_adapter.go @@ -0,0 +1,110 @@ +/* + * Copyright 2024-2025 KusionStack Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package revisionadapter + +import ( + "context" + "encoding/json" + + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "kusionstack.io/kube-utils/xset/api" + "kusionstack.io/kube-utils/xset/xcontrol" +) + +type RevisionOwnerAdapter struct { + api.XSetController + + xcontrol.TargetControl +} + +func NewRevisionOwnerAdapter(xsetController api.XSetController, xcontrol xcontrol.TargetControl) *RevisionOwnerAdapter { + return &RevisionOwnerAdapter{ + XSetController: xsetController, + TargetControl: xcontrol, + } +} + +func (r *RevisionOwnerAdapter) GetSelector(obj metav1.Object) *metav1.LabelSelector { + xset := obj.(api.XSetObject) + return r.XSetController.GetXSetSpec(xset).Selector +} + +func (r *RevisionOwnerAdapter) GetCollisionCount(obj metav1.Object) *int32 { + xset := obj.(api.XSetObject) + return r.XSetController.GetXSetStatus(xset).CollisionCount +} + +func (r *RevisionOwnerAdapter) GetHistoryLimit(obj metav1.Object) int32 { + xset := obj.(api.XSetObject) + return r.XSetController.GetXSetSpec(xset).HistoryLimit +} + +func (r *RevisionOwnerAdapter) GetPatch(obj metav1.Object) ([]byte, error) { + return r.getXSetPatch(obj) +} + +func (r *RevisionOwnerAdapter) GetCurrentRevision(obj metav1.Object) string { + xset := obj.(api.XSetObject) + return r.XSetController.GetXSetStatus(xset).CurrentRevision +} + +func (r *RevisionOwnerAdapter) IsInUsed(obj metav1.Object, revision string) bool { + xSetObject, _ := obj.(api.XSetObject) + spec := r.XSetController.GetXSetSpec(xSetObject) + status := r.XSetController.GetXSetStatus(xSetObject) + + if status.UpdatedRevision == revision || status.CurrentRevision == revision { + return true + } + + targets, _ := r.TargetControl.GetFilteredTargets(context.TODO(), spec.Selector, xSetObject) + for _, target := range targets { + if target.GetLabels() != nil { + currentRevisionName, exist := target.GetLabels()[appsv1.ControllerRevisionHashLabelKey] + if exist && currentRevisionName == revision { + return true + } + } + } + + return false +} + +func (r *RevisionOwnerAdapter) getXSetPatch(obj metav1.Object) ([]byte, error) { + xset := obj.(api.XSetObject) + xTpl := r.XSetController.GetXTemplate(xset) + tplBytes, err := json.Marshal(xTpl) + if err != nil { + return nil, err + } + tplMap := make(map[string]interface{}) + err = json.Unmarshal(tplBytes, &tplMap) + if err != nil { + return nil, err + } + + tplMap["$patch"] = "replace" + specMap := make(map[string]interface{}) + specMap["template"] = tplMap + objMap := make(map[string]interface{}) + objMap["spec"] = specMap + + patch, err := json.Marshal(objMap) + return patch, err +} diff --git a/xset/synccontrols/const.go b/xset/synccontrols/const.go new file mode 100644 index 00000000..70edd4b4 --- /dev/null +++ b/xset/synccontrols/const.go @@ -0,0 +1,46 @@ +/* + * Copyright 2024-2025 KusionStack Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package synccontrols + +// XSet labels +const ( + // TargetInstanceIDLabelKey is used to attach target instance ID on target + TargetInstanceIDLabelKey = "xset.kusionstack.io/instance-id" + // XSetUpdateIndicateLabelKey is used to indicate a target should be updated by label + XSetUpdateIndicateLabelKey = "xset.kusionstack.io/update-included" + + // TargetDeletionIndicationLabelKey indicates a target will be deleted by xset + TargetDeletionIndicationLabelKey = "xset.kusionstack.io/to-delete" + // TargetReplaceIndicationLabelKey indicates a target will be replaced by xset + TargetReplaceIndicationLabelKey = "xset.kusionstack.io/to-replace" + // TargetReplaceByReplaceUpdateLabelKey indicates a target is replaced by update by xset + TargetReplaceByReplaceUpdateLabelKey = "xset.kusionstack.io/replaced-by-replace-update" + // TargetExcludeIndicationLabelKey indicates a target will be excluded by xset + TargetExcludeIndicationLabelKey = "xset.kusionstack.io/to-exclude" + + // TargetReplacePairOriginName is used to indicate replace origin target name on the new created target + TargetReplacePairOriginName = "xset.kusionstack.io/replace-pair-origin-name" + // TargetReplacePairNewId is used to indicate the new created target instance on replace origin target + TargetReplacePairNewId = "xset.kusionstack.io/replace-pair-new-id" + + // TargetOrphanedIndicateLabelKey indicates target is orphaned + TargetOrphanedIndicateLabelKey = "xset.kusionstack.io/orphaned" +) + +const ( + LastTargetStatusAnnotationKey = "xset.kusionstack.io/last-target-status" +) diff --git a/xset/synccontrols/inexclude.go b/xset/synccontrols/inexclude.go new file mode 100644 index 00000000..1f1aa4fc --- /dev/null +++ b/xset/synccontrols/inexclude.go @@ -0,0 +1,64 @@ +/* + * Copyright 2024-2025 KusionStack Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package synccontrols + +import ( + "fmt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" +) + +// AllowResourceExclude checks if pod or pvc is allowed to exclude +func AllowResourceExclude(obj metav1.Object, ownerName, ownerKind string) (bool, string) { + labels := obj.GetLabels() + // not controlled by ks manager + if labels == nil { + return false, "object's label is empty" + } else if val, exist := labels[appsv1alpha1.ControlledByKusionStackLabelKey]; !exist || val != "true" { + return false, "object is not controlled by kusionstack system" + } + + // not controlled by current xset + if controller := metav1.GetControllerOf(obj); controller == nil || controller.Name != ownerName || controller.Kind != ownerKind { + return false, "object is not owned by any one, not allowed to exclude" + } + return true, "" +} + +// AllowResourceInclude checks if pod or pvc is allowed to include +func AllowResourceInclude(obj metav1.Object, ownerName, ownerKind string) (bool, string) { + labels := obj.GetLabels() + ownerRefs := obj.GetOwnerReferences() + + // not controlled by ks manager + if labels == nil { + return false, "object's label is empty" + } else if val, exist := labels[appsv1alpha1.ControlledByKusionStackLabelKey]; !exist || val != "true" { + return false, "object is not controlled by kusionstack system" + } + + if ownerRefs != nil { + if controller := metav1.GetControllerOf(obj); controller != nil { + // controlled by others + if controller.Name != ownerName || controller.Kind != ownerKind { + return false, fmt.Sprintf("object's ownerReference controller is not %s/%s", ownerKind, ownerName) + } + } + } + return true, "" +} diff --git a/xset/synccontrols/inexclude_test.go b/xset/synccontrols/inexclude_test.go new file mode 100644 index 00000000..7241b326 --- /dev/null +++ b/xset/synccontrols/inexclude_test.go @@ -0,0 +1,300 @@ +/* + * Copyright 2024-2025 KusionStack Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package synccontrols + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" +) + +func TestAllowResourceExclude(t *testing.T) { + ownerName, ownerKind := "test", "CollaSet" + tests := []struct { + name string + obj *corev1.Pod + allow bool + reason string + }{ + { + name: "label is nil", + obj: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: nil, + }, + }, + allow: false, + reason: "object's label is empty", + }, + { + name: "KusionStack control label not satisfied", + obj: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + appsv1alpha1.ControlledByKusionStackLabelKey: "false", + }, + }, + }, + allow: false, + reason: "object is not controlled by kusionstack system", + }, + { + name: "controller is nil", + obj: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + appsv1alpha1.ControlledByKusionStackLabelKey: "true", + }, + OwnerReferences: make([]metav1.OwnerReference, 0), + }, + }, + allow: false, + reason: "object is not owned by any one, not allowed to exclude", + }, + { + name: "controller name not equals to ownerName", + obj: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + appsv1alpha1.ControlledByKusionStackLabelKey: "true", + }, + OwnerReferences: []metav1.OwnerReference{ + { + Name: "test1", + Kind: ownerKind, + Controller: pointer.Bool(true), + }, + }, + }, + }, + allow: false, + reason: "object is not owned by any one, not allowed to exclude", + }, + { + name: "controller kind not equals to ownerName", + obj: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + appsv1alpha1.ControlledByKusionStackLabelKey: "true", + }, + OwnerReferences: []metav1.OwnerReference{ + { + Name: ownerName, + Kind: "kind2", + Controller: pointer.Bool(true), + }, + }, + }, + }, + allow: false, + reason: "object is not owned by any one, not allowed to exclude", + }, + { + name: "allowed case", + obj: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + appsv1alpha1.ControlledByKusionStackLabelKey: "true", + }, + OwnerReferences: []metav1.OwnerReference{ + { + Name: ownerName, + Kind: ownerKind, + Controller: pointer.Bool(true), + }, + }, + }, + }, + allow: true, + reason: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1 := AllowResourceExclude(tt.obj, ownerName, ownerKind) + if got != tt.allow { + t.Errorf("AllowResourceExclude() got = %v, want %v", got, tt.allow) + } + if got1 != tt.reason { + t.Errorf("AllowResourceExclude() got1 = %v, want %v", got1, tt.reason) + } + }) + } +} + +func TestAllowResourceInclude(t *testing.T) { + ownerName := "test" + ownerKind := "CollaSet" + tests := []struct { + name string + obj *corev1.Pod + allow bool + reason string + }{ + { + name: "label is nil", + obj: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: nil, + }, + }, + allow: false, + reason: "object's label is empty", + }, + { + name: "KusionStack control label not satisfied", + obj: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + appsv1alpha1.ControlledByKusionStackLabelKey: "false", + }, + }, + }, + allow: false, + reason: "object is not controlled by kusionstack system", + }, + { + name: "controller is nil", + obj: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + appsv1alpha1.ControlledByKusionStackLabelKey: "true", + }, + OwnerReferences: make([]metav1.OwnerReference, 0), + }, + }, + allow: true, + reason: "", + }, + { + name: "controller name not equals to ownerName", + obj: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + appsv1alpha1.ControlledByKusionStackLabelKey: "true", + }, + OwnerReferences: []metav1.OwnerReference{ + { + Name: "test1", + Kind: ownerKind, + Controller: pointer.Bool(true), + }, + }, + }, + }, + allow: false, + reason: "object's ownerReference controller is not CollaSet/test", + }, + { + name: "controller kind not equals to ownerName", + obj: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + appsv1alpha1.ControlledByKusionStackLabelKey: "true", + }, + OwnerReferences: []metav1.OwnerReference{ + { + Name: "test", + Kind: "kind2", + Controller: pointer.Bool(true), + }, + }, + }, + }, + allow: false, + reason: "object's ownerReference controller is not CollaSet/test", + }, + { + name: "controller kind not equals to ownerName", + obj: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + appsv1alpha1.ControlledByKusionStackLabelKey: "true", + }, + OwnerReferences: []metav1.OwnerReference{ + { + Name: ownerName, + Kind: ownerKind, + Controller: pointer.Bool(true), + }, + }, + }, + }, + allow: true, + reason: "", + }, + { + name: "allowed case1", + obj: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + appsv1alpha1.ControlledByKusionStackLabelKey: "true", + appsv1alpha1.PodOrphanedIndicateLabelKey: "true", + }, + OwnerReferences: []metav1.OwnerReference{ + { + Name: ownerName, + Kind: ownerKind, + Controller: pointer.Bool(true), + }, + }, + }, + }, + allow: true, + reason: "", + }, + { + name: "allowed case2", + obj: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + appsv1alpha1.ControlledByKusionStackLabelKey: "true", + }, + }, + }, + allow: true, + reason: "", + }, + { + name: "allowed case3", + obj: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + appsv1alpha1.ControlledByKusionStackLabelKey: "true", + appsv1alpha1.PodOrphanedIndicateLabelKey: "true", + }, + }, + }, + allow: true, + reason: "", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1 := AllowResourceInclude(tt.obj, ownerName, ownerKind) + if got != tt.allow { + t.Errorf("AllowResourceExclude() got = %v, want %v", got, tt.allow) + } + if got1 != tt.reason { + t.Errorf("AllowResourceExclude() got1 = %v, want %v", got1, tt.reason) + } + }) + } +} diff --git a/xset/synccontrols/sync_control.go b/xset/synccontrols/sync_control.go new file mode 100644 index 00000000..d22e7d3d --- /dev/null +++ b/xset/synccontrols/sync_control.go @@ -0,0 +1,911 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package synccontrols + +import ( + "context" + "fmt" + "strconv" + "strings" + "sync/atomic" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/util/retry" + appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "kusionstack.io/kube-utils/controller/expectations" + "kusionstack.io/kube-utils/controller/mixin" + controllerutils "kusionstack.io/kube-utils/controller/utils" + "kusionstack.io/kube-utils/xset/api" + "kusionstack.io/kube-utils/xset/opslifecycle" + "kusionstack.io/kube-utils/xset/resourcecontexts" + "kusionstack.io/kube-utils/xset/xcontrol" +) + +type SyncControl interface { + SyncTargets(ctx context.Context, instance api.XSetObject, syncContext *SyncContext) (bool, error) + + Replace(ctx context.Context, instance api.XSetObject, syncContext *SyncContext) error + + Scale(ctx context.Context, instance api.XSetObject, syncContext *SyncContext) (bool, *time.Duration, error) + + Update(ctx context.Context, instance api.XSetObject, syncContext *SyncContext) (bool, *time.Duration, error) + + CalculateStatus(ctx context.Context, instance api.XSetObject, syncContext *SyncContext) *api.XSetStatus +} + +func NewRealSyncControl(reconcileMixIn *mixin.ReconcilerMixin, + xsetController api.XSetController, + xControl xcontrol.TargetControl, + cacheExpectation *expectations.CacheExpectation, +) SyncControl { + scaleInOpsLicecycleAdapter := xsetController.GetScaleInOpsLifecycleAdapter() + if scaleInOpsLicecycleAdapter == nil { + scaleInOpsLicecycleAdapter = &opslifecycle.DefaultScaleInLifecycleAdapter{} + } + updateLifecycleAdapter := xsetController.GetUpdateOpsLifecycleAdapter() + if updateLifecycleAdapter == nil { + updateLifecycleAdapter = &opslifecycle.DefaultUpdateLifecycleAdapter{} + } + + xMeta := xsetController.XMeta() + targetGVK := xMeta.GroupVersionKind() + xsetMeta := xsetController.XSetMeta() + xsetGVK := xsetMeta.GroupVersionKind() + + updateConfig := &UpdateConfig{ + xsetController: xsetController, + client: reconcileMixIn.Client, + targetControl: xControl, + recorder: reconcileMixIn.Recorder, + + scaleInLifecycleAdapter: scaleInOpsLicecycleAdapter, + updateLifecycleAdapter: updateLifecycleAdapter, + cacheExpectation: cacheExpectation, + targetGVK: targetGVK, + } + return &RealSyncControl{ + ReconcilerMixin: *reconcileMixIn, + xsetController: xsetController, + xControl: xControl, + + updateConfig: updateConfig, + cacheExpectation: cacheExpectation, + xsetGVK: xsetGVK, + targetGVK: targetGVK, + + scaleInLifecycleAdapter: scaleInOpsLicecycleAdapter, + updateLifecycleAdapter: updateLifecycleAdapter, + } +} + +var _ SyncControl = &RealSyncControl{} + +type RealSyncControl struct { + mixin.ReconcilerMixin + xControl xcontrol.TargetControl + xsetController api.XSetController + + updateConfig *UpdateConfig + scaleInLifecycleAdapter api.LifecycleAdapter + updateLifecycleAdapter api.LifecycleAdapter + + cacheExpectation *expectations.CacheExpectation + xsetGVK schema.GroupVersionKind + targetGVK schema.GroupVersionKind +} + +// SyncTargets is used to parse targetWrappers and reclaim Target instance ID +func (r *RealSyncControl) SyncTargets(ctx context.Context, instance api.XSetObject, syncContext *SyncContext) ( + bool, error, +) { + xspec := r.xsetController.GetXSetSpec(instance) + if xspec == nil { + return false, fmt.Errorf("fail to get XSetSpec") + } + + var err error + syncContext.FilteredTarget, err = r.xControl.GetFilteredTargets(ctx, xspec.Selector, instance) + if err != nil { + return false, fmt.Errorf("fail to get filtered Targets: %w", err) + } + + toExcludeTargetNames, toIncludeTargetNames, err := r.dealIncludeExcludeTargets(ctx, instance, syncContext.FilteredTarget) + if err != nil { + return false, fmt.Errorf("fail to deal with include exclude targets: %s", err.Error()) + } + + // get owned IDs + var ownedIDs map[int]*appsv1alpha1.ContextDetail + if err = retry.RetryOnConflict(retry.DefaultRetry, func() error { + ownedIDs, err = resourcecontexts.AllocateID(r.xsetController, r.Client, r.cacheExpectation, instance, syncContext.UpdatedRevision.GetName(), int(RealValue(xspec.Replicas))) + syncContext.OwnedIds = ownedIDs + return err + }); err != nil { + return false, fmt.Errorf("fail to allocate %d IDs using context when sync Targets: %w", xspec.Replicas, err) + } + + // stateless case + var targetWrappers []targetWrapper + syncContext.CurrentIDs = sets.Int{} + idToReclaim := sets.Int{} + toDeleteTargetNames := sets.NewString(xspec.ScaleStrategy.TargetToDelete...) + + for i := range syncContext.FilteredTarget { + target := syncContext.FilteredTarget[i] + xName := target.GetName() + id, _ := GetInstanceID(target) + toDelete := toDeleteTargetNames.Has(xName) + toExclude := toExcludeTargetNames.Has(xName) + + // priority: toDelete > toReplace > toExclude + if toDelete { + toDeleteTargetNames.Delete(xName) + } + if toExclude { + if targetDuringReplace(target) || toDelete { + // skip exclude until replace and toDelete done + toExcludeTargetNames.Delete(xName) + } else { + // exclude target and delete its targetContext + idToReclaim.Insert(id) + } + } + + if target.GetDeletionTimestamp() != nil { + // 1. Reclaim ID from Target which is scaling in and terminating. + if contextDetail, exist := ownedIDs[id]; exist && contextDetail.Contains(resourcecontexts.ScaleInContextDataKey, "true") { + idToReclaim.Insert(id) + } + + _, replaceIndicate := target.GetLabels()[TargetReplaceIndicationLabelKey] + // 2. filter out Targets which are terminating and not replace indicate + if !replaceIndicate { + continue + } + } + + targetWrappers = append(targetWrappers, targetWrapper{ + Object: target, + ID: id, + ContextDetail: ownedIDs[id], + PlaceHolder: false, + + ToDelete: toDelete, + ToExclude: toExclude, + + IsDuringScaleInOps: opslifecycle.IsDuringOps(r.scaleInLifecycleAdapter, target), + IsDuringUpdateOps: opslifecycle.IsDuringOps(r.updateLifecycleAdapter, target), + }) + + if id >= 0 { + syncContext.CurrentIDs.Insert(id) + } + } + + // do include exclude targets, and skip doSync() if succeeded + var inExSucceed bool + if len(toExcludeTargetNames) > 0 || len(toIncludeTargetNames) > 0 { + var availableContexts []*appsv1alpha1.ContextDetail + var getErr error + availableContexts, ownedIDs, getErr = r.getAvailableTargetIDs(len(toIncludeTargetNames), instance, syncContext) + if getErr != nil { + return false, getErr + } + if err = r.doIncludeExcludeTargets(ctx, instance, toExcludeTargetNames.List(), toIncludeTargetNames.List(), availableContexts); err != nil { + r.Recorder.Eventf(instance, corev1.EventTypeWarning, "ExcludeIncludeFailed", "%s syncTargets include exclude with error: %s", r.xsetGVK.Kind, err.Error()) + return false, err + } + inExSucceed = true + } + + // reclaim Target ID which is (1) during ScalingIn, (2) ExcludeTargets + err = r.reclaimOwnedIDs(false, instance, idToReclaim, ownedIDs, syncContext.CurrentIDs) + if err != nil { + r.Recorder.Eventf(instance, corev1.EventTypeWarning, "ReclaimOwnedIDs", "reclaim target contexts with error: %s", err.Error()) + return false, err + } + + // reclaim scaleStrategy for delete, exclude, include + err = r.reclaimScaleStrategy(ctx, toDeleteTargetNames, toExcludeTargetNames, toIncludeTargetNames, instance) + if err != nil { + r.Recorder.Eventf(instance, corev1.EventTypeWarning, "ReclaimScaleStrategy", "reclaim scaleStrategy with error: %s", err.Error()) + return false, err + } + + syncContext.TargetWrappers = targetWrappers + syncContext.OwnedIds = ownedIDs + + syncContext.activeTargets = FilterOutActiveTargetWrappers(syncContext.TargetWrappers) + syncContext.replacingMap = classifyTargetReplacingMapping(syncContext.activeTargets) + + return inExSucceed, nil +} + +// dealIncludeExcludeTargets returns targets which are allowed to exclude and include +func (r *RealSyncControl) dealIncludeExcludeTargets(ctx context.Context, xsetObject api.XSetObject, targets []client.Object) (sets.String, sets.String, error) { + spec := r.xsetController.GetXSetSpec(xsetObject) + ownedTargets := sets.String{} + excludeTargetNames := sets.String{} + includeTargetNames := sets.String{} + + for _, target := range targets { + ownedTargets.Insert(target.GetName()) + if _, exist := target.GetLabels()[TargetExcludeIndicationLabelKey]; exist { + excludeTargetNames.Insert(target.GetName()) + } + } + + tmpUnOwnedExcludeTargets := sets.String{} + for _, targetName := range spec.ScaleStrategy.TargetToExclude { + if ownedTargets.Has(targetName) { + excludeTargetNames.Insert(targetName) + } else { + tmpUnOwnedExcludeTargets.Insert(targetName) + } + } + + intersection := sets.String{} + for _, targetName := range spec.ScaleStrategy.TargetToInclude { + if excludeTargetNames.Has(targetName) { + intersection.Insert(targetName) + excludeTargetNames.Delete(targetName) + } else if tmpUnOwnedExcludeTargets.Has(targetName) { + intersection.Insert(targetName) + } else if !ownedTargets.Has(targetName) { + includeTargetNames.Insert(targetName) + } + } + + if len(intersection) > 0 { + r.Recorder.Eventf(xsetObject, corev1.EventTypeWarning, "DupExIncludedTarget", "duplicated targets %s in both excluding and including sets", strings.Join(intersection.List(), ", ")) + } + + // seem no need to check allow ResourceExclude, since filterTargets already filter only owned targets + toExcludeTargets, notAllowedExcludeTargets, exErr := r.allowIncludeExcludeTargets(ctx, xsetObject, excludeTargetNames.List(), AllowResourceExclude) + toIncludeTargets, notAllowedIncludeTargets, inErr := r.allowIncludeExcludeTargets(ctx, xsetObject, includeTargetNames.List(), AllowResourceInclude) + if notAllowedExcludeTargets.Len() > 0 { + r.Recorder.Eventf(xsetObject, corev1.EventTypeWarning, "ExcludeNotAllowed", fmt.Sprintf("targets [%v] are not allowed to exclude, please find out the reason from target's event", notAllowedExcludeTargets.List())) + } + if notAllowedIncludeTargets.Len() > 0 { + r.Recorder.Eventf(xsetObject, corev1.EventTypeWarning, "IncludeNotAllowed", fmt.Sprintf("targets [%v] are not allowed to include, please find out the reason from target's event", notAllowedIncludeTargets.List())) + } + return toExcludeTargets, toIncludeTargets, controllerutils.AggregateErrors([]error{exErr, inErr}) +} + +// checkAllowFunc refers to AllowResourceExclude and AllowResourceInclude +type checkAllowFunc func(obj metav1.Object, ownerName, ownerKind string) (bool, string) + +// allowIncludeExcludeTargets try to classify targetNames to allowedTargets and notAllowedTargets, using checkAllowFunc func +func (r *RealSyncControl) allowIncludeExcludeTargets(ctx context.Context, xset api.XSetObject, targetNames []string, fn checkAllowFunc) (allowTargets, notAllowTargets sets.String, err error) { + allowTargets = sets.String{} + notAllowTargets = sets.String{} + for i := range targetNames { + target := r.xsetController.EmptyXObject() + targetName := targetNames[i] + err = r.Client.Get(ctx, types.NamespacedName{Namespace: xset.GetNamespace(), Name: targetName}, target) + if errors.IsNotFound(err) { + notAllowTargets.Insert(targetNames[i]) + continue + } else if err != nil { + r.Recorder.Eventf(xset, corev1.EventTypeWarning, "ExcludeIncludeFailed", fmt.Sprintf("failed to find target %s: %s", targetNames[i], err.Error())) + return + } + + // check allowance for target + if allowed, reason := fn(target, xset.GetName(), xset.GetObjectKind().GroupVersionKind().Kind); !allowed { + r.Recorder.Eventf(target, corev1.EventTypeWarning, "ExcludeIncludeNotAllowed", + fmt.Sprintf("target is not allowed to exclude/include from/to %s %s/%s: %s", r.xsetGVK.Kind, xset.GetNamespace(), xset.GetName(), reason)) + notAllowTargets.Insert(targetName) + continue + } + allowTargets.Insert(targetName) + } + return allowTargets, notAllowTargets, nil +} + +// Replace is used to replace replace-indicate targets +func (r *RealSyncControl) Replace(ctx context.Context, xsetObject api.XSetObject, syncContext *SyncContext) error { + var err error + var needUpdateContext bool + var idToReclaim sets.Int + + needReplaceOriginTargets, needCleanLabelTargets, targetsNeedCleanLabels, needDeleteTargets := r.dealReplaceTargets(syncContext.FilteredTarget) + + // delete origin targets for replace + err = BatchDelete(ctx, r.xControl, needDeleteTargets) + if err != nil { + r.Recorder.Eventf(xsetObject, corev1.EventTypeWarning, "ReplaceTarget", "delete targets by label with error: %s", err.Error()) + return err + } + + // clean labels for replace targets + needUpdateContext, idToReclaim, err = r.cleanReplaceTargetLabels(ctx, needCleanLabelTargets, targetsNeedCleanLabels, syncContext.OwnedIds, syncContext.CurrentIDs) + if err != nil { + r.Recorder.Eventf(xsetObject, corev1.EventTypeWarning, "ReplaceTarget", fmt.Sprintf("clean targets replace pair origin name label with error: %s", err.Error())) + return err + } + + // create new targets for need replace targets + if len(needReplaceOriginTargets) > 0 { + var availableContexts []*appsv1alpha1.ContextDetail + var getErr error + availableContexts, syncContext.OwnedIds, getErr = r.getAvailableTargetIDs(len(needReplaceOriginTargets), xsetObject, syncContext) + if getErr != nil { + return getErr + } + successCount, err := r.replaceOriginTargets(ctx, xsetObject, syncContext, needReplaceOriginTargets, syncContext.OwnedIds, availableContexts) + needUpdateContext = needUpdateContext || successCount > 0 + if err != nil { + r.Recorder.Eventf(xsetObject, corev1.EventTypeWarning, "ReplaceTarget", "deal replace targets with error: %s", err.Error()) + return err + } + } + + // reclaim Target ID which is ReplaceOriginTarget + if err := r.reclaimOwnedIDs(needUpdateContext, xsetObject, idToReclaim, syncContext.OwnedIds, syncContext.CurrentIDs); err != nil { + r.Recorder.Eventf(xsetObject, corev1.EventTypeWarning, "ReclaimOwnedIDs", "reclaim target contexts with error: %s", err.Error()) + return err + } + + // create targetWrappers for non-exist targets + for id, contextDetail := range syncContext.OwnedIds { + if _, inUsed := syncContext.CurrentIDs[id]; inUsed { + continue + } + syncContext.TargetWrappers = append(syncContext.TargetWrappers, targetWrapper{ + ID: id, + Object: nil, + ContextDetail: contextDetail, + PlaceHolder: true, + }) + } + + return nil +} + +func (r *RealSyncControl) Scale(ctx context.Context, xsetObject api.XSetObject, syncContext *SyncContext) (bool, *time.Duration, error) { + spec := r.xsetController.GetXSetSpec(xsetObject) + logger := r.Logger.WithValues(r.xsetGVK.Kind, ObjectKeyString(xsetObject)) + var recordedRequeueAfter *time.Duration + + diff := int(RealValue(spec.Replicas)) - len(syncContext.replacingMap) + scaling := false + + if diff >= 0 { + // trigger delete targets indicated in ScaleStrategy.TargetToDelete by label + for _, targetWrapper := range syncContext.activeTargets { + if targetWrapper.ToDelete { + err := BatchDelete(ctx, r.xControl, []client.Object{targetWrapper.Object}) + if err != nil { + return false, recordedRequeueAfter, err + } + } + } + + // scale out targets and return if diff > 0 + if diff > 0 { + // collect instance ID in used from owned Targets + targetInstanceIDSet := sets.Int{} + for _, target := range syncContext.activeTargets { + targetInstanceIDSet[target.ID] = struct{}{} + } + + // find IDs and their contexts which have not been used by owned Targets + availableContext := resourcecontexts.ExtractAvailableContexts(diff, syncContext.OwnedIds, targetInstanceIDSet) + needUpdateContext := atomic.Bool{} + succCount, err := controllerutils.SlowStartBatch(len(availableContext), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) (err error) { + availableIDContext := availableContext[i] + defer func() { + if decideContextRevision(availableIDContext, syncContext.UpdatedRevision, err == nil) { + needUpdateContext.Store(true) + } + }() + // use revision recorded in Context + revision := syncContext.UpdatedRevision + if revisionName, exist := availableIDContext.Data[resourcecontexts.RevisionContextDataKey]; exist && revisionName != "" { + for i := range syncContext.Revisions { + if syncContext.Revisions[i].GetName() == revisionName { + revision = syncContext.Revisions[i] + break + } + } + } + // scale out new Targets with updatedRevision + // TODO use cache + target, err := NewTargetFrom(r.xsetController, xsetObject, revision, availableIDContext.ID) + + labels := target.GetLabels() + if availableIDContext.Data[resourcecontexts.JustCreateContextDataKey] == "true" { + labels[opslifecycle.TargetCreatingLabel] = strconv.FormatInt(time.Now().UnixNano(), 10) + } else { + labels[opslifecycle.TargetCompletingLabel] = strconv.FormatInt(time.Now().UnixNano(), 10) + } + + if err != nil { + return fmt.Errorf("fail to new Target from revision %s: %w", revision.GetName(), err) + } + newTarget := target.DeepCopyObject().(client.Object) + logger.V(1).Info("try to create Target with revision of "+r.xsetGVK.Kind, "revision", revision.GetName()) + if target, err = r.xControl.CreateTarget(ctx, newTarget); err != nil { + return err + } + // add an expectation for this target creation, before next reconciling + return r.cacheExpectation.ExpectCreation(r.targetGVK, target.GetNamespace(), target.GetName()) + }) + if needUpdateContext.Load() { + logger.V(1).Info("try to update ResourceContext for XSet after scaling out") + if updateContextErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { + return resourcecontexts.UpdateToTargetContext(r.xsetController, r.Client, r.cacheExpectation, xsetObject, syncContext.OwnedIds) + }); updateContextErr != nil { + err = controllerutils.AggregateErrors([]error{updateContextErr, err}) + } + } + if err != nil { + AddOrUpdateCondition(syncContext.NewStatus, api.XSetScale, err, "ScaleOutFailed", err.Error()) + return succCount > 0, recordedRequeueAfter, err + } + r.Recorder.Eventf(xsetObject, corev1.EventTypeNormal, "ScaleOut", "scale out %d Target(s)", succCount) + AddOrUpdateCondition(syncContext.NewStatus, api.XSetScale, nil, "ScaleOut", "") + return succCount > 0, recordedRequeueAfter, err + } else { + AddOrUpdateCondition(syncContext.NewStatus, api.XSetScale, nil, "ScaleOut", "") + return false, nil, nil + } + } else if diff < 0 { + // chose the targets to scale in + targetsToScaleIn := r.getTargetsToDelete(syncContext.activeTargets, syncContext.replacingMap, diff*-1) + // filter out Targets need to trigger TargetOpsLifecycle + wrapperCh := make(chan *targetWrapper, len(targetsToScaleIn)) + for i := range targetsToScaleIn { + if targetsToScaleIn[i].IsDuringScaleInOps { + continue + } + wrapperCh <- targetsToScaleIn[i] + } + + // trigger Targets to enter TargetOpsLifecycle + succCount, err := controllerutils.SlowStartBatch(len(wrapperCh), controllerutils.SlowStartInitialBatchSize, false, func(_ int, err error) error { + wrapper := <-wrapperCh + object := wrapper.Object + + // trigger TargetOpsLifecycle with scaleIn OperationType + logger.V(1).Info("try to begin TargetOpsLifecycle for scaling in Target in XSet", "wrapper", ObjectKeyString(object)) + // todo switch to x opslifecycle + if updated, err := opslifecycle.Begin(r.Client, r.scaleInLifecycleAdapter, object); err != nil { + return fmt.Errorf("fail to begin TargetOpsLifecycle for Scaling in Target %s/%s: %w", object.GetNamespace(), object.GetName(), err) + } else if updated { + r.Recorder.Eventf(object, corev1.EventTypeNormal, "BeginScaleInLifecycle", "succeed to begin TargetOpsLifecycle for scaling in") + // add an expectation for this wrapper creation, before next reconciling + if err := r.cacheExpectation.ExpectUpdation(r.targetGVK, object.GetNamespace(), object.GetName(), object.GetResourceVersion()); err != nil { + return err + } + } + + return nil + }) + scaling = succCount != 0 + + if err != nil { + AddOrUpdateCondition(syncContext.NewStatus, api.XSetScale, err, "ScaleInFailed", err.Error()) + return scaling, recordedRequeueAfter, err + } else { + AddOrUpdateCondition(syncContext.NewStatus, api.XSetScale, nil, "ScaleIn", "") + } + + needUpdateContext := false + for i, targetWrapper := range targetsToScaleIn { + requeueAfter, allowed := opslifecycle.AllowOps(r.scaleInLifecycleAdapter, RealValue(spec.ScaleStrategy.OperationDelaySeconds), targetWrapper.Object) + if !allowed && targetWrapper.Object.GetDeletionTimestamp() == nil { + r.Recorder.Eventf(targetWrapper.Object, corev1.EventTypeNormal, "TargetScaleInLifecycle", "Target is not allowed to scale in") + continue + } + + if requeueAfter != nil { + r.Recorder.Eventf(targetWrapper.Object, corev1.EventTypeNormal, "TargetScaleInLifecycle", "delay Target scale in for %d seconds", requeueAfter.Seconds()) + if recordedRequeueAfter == nil || *requeueAfter < *recordedRequeueAfter { + recordedRequeueAfter = requeueAfter + } + + continue + } + + // if Target is allowed to operate or Target has already been deleted, promte to delete Target + if contextDetail, exist := syncContext.OwnedIds[targetWrapper.ID]; exist && !contextDetail.Contains(resourcecontexts.ScaleInContextDataKey, "true") { + needUpdateContext = true + contextDetail.Put(resourcecontexts.ScaleInContextDataKey, "true") + } + + if targetWrapper.GetDeletionTimestamp() != nil { + continue + } + + wrapperCh <- targetsToScaleIn[i] + } + + // mark these Targets to scalingIn + if needUpdateContext { + logger.V(1).Info("try to update ResourceContext for XSet when scaling in Target") + if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + return resourcecontexts.UpdateToTargetContext(r.xsetController, r.Client, r.cacheExpectation, xsetObject, syncContext.OwnedIds) + }); err != nil { + AddOrUpdateCondition(syncContext.NewStatus, api.XSetScale, err, "ScaleInFailed", fmt.Sprintf("failed to update Context for scaling in: %s", err)) + return scaling, recordedRequeueAfter, err + } else { + AddOrUpdateCondition(syncContext.NewStatus, api.XSetScale, nil, "ScaleIn", "") + } + } + + // do delete Target resource + succCount, err = controllerutils.SlowStartBatch(len(wrapperCh), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) error { + target := <-wrapperCh + logger.V(1).Info("try to scale in Target", "target", ObjectKeyString(target)) + if err := r.xControl.DeleteTarget(ctx, target.Object); err != nil { + return fmt.Errorf("fail to delete Target %s/%s when scaling in: %w", target.GetNamespace(), target.GetName(), err) + } + + r.Recorder.Eventf(xsetObject, corev1.EventTypeNormal, "TargetDeleted", "succeed to scale in Target %s/%s", target.GetNamespace(), target.GetName()) + return nil + }) + scaling = scaling || succCount > 0 + + if succCount > 0 { + r.Recorder.Eventf(xsetObject, corev1.EventTypeNormal, "ScaleIn", "scale in %d Target(s)", succCount) + } + if err != nil { + AddOrUpdateCondition(syncContext.NewStatus, api.XSetScale, err, "ScaleInFailed", fmt.Sprintf("fail to delete Target for scaling in: %s", err)) + return scaling, recordedRequeueAfter, err + } else { + AddOrUpdateCondition(syncContext.NewStatus, api.XSetScale, nil, "ScaleIn", "") + } + + return scaling, recordedRequeueAfter, err + } + + // reset ContextDetail.ScalingIn, if there are Targets had its TargetOpsLifecycle reverted + needUpdateTargetContext := false + for _, targetWrapper := range syncContext.activeTargets { + if contextDetail, exist := syncContext.OwnedIds[targetWrapper.ID]; exist && contextDetail.Contains(resourcecontexts.ScaleInContextDataKey, "true") && + !opslifecycle.IsDuringOps(r.scaleInLifecycleAdapter, targetWrapper) { + needUpdateTargetContext = true + contextDetail.Remove(resourcecontexts.ScaleInContextDataKey) + } + } + + if needUpdateTargetContext { + logger.V(1).Info("try to update ResourceContext for XSet after scaling") + if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + return resourcecontexts.UpdateToTargetContext(r.xsetController, r.Client, r.cacheExpectation, xsetObject, syncContext.OwnedIds) + }); err != nil { + return scaling, recordedRequeueAfter, fmt.Errorf("fail to reset ResourceContext: %w", err) + } + } + + return scaling, recordedRequeueAfter, nil +} + +func (r *RealSyncControl) Update(ctx context.Context, xsetObject api.XSetObject, syncContext *SyncContext) (bool, *time.Duration, error) { + logger := r.Logger.WithValues("xset", ObjectKeyString(xsetObject)) + var err error + var recordedRequeueAfter *time.Duration + // 1. scan and analysis targets update info for active targets and PlaceHolder targets + targetUpdateInfos := r.attachTargetUpdateInfo(xsetObject, syncContext) + + // 2. decide Target update candidates + candidates := decideTargetToUpdate(r.xsetController, xsetObject, targetUpdateInfos) + targetToUpdate := filterOutPlaceHolderUpdateInfos(candidates) + targetCh := make(chan *targetUpdateInfo, len(targetToUpdate)) + updater := r.newTargetUpdater(xsetObject) + updating := false + + // 3. filter already updated revision, + for i, targetInfo := range targetToUpdate { + if targetInfo.IsUpdatedRevision { + continue + } + + // 3.1 fulfillTargetUpdateInfo to all not updatedRevision target + if targetInfo.CurrentRevision.GetName() != UnknownRevision { + if err = updater.FulfillTargetUpdatedInfo(ctx, syncContext.UpdatedRevision, targetInfo); err != nil { + logger.Error(err, fmt.Sprintf("fail to analyze target %s/%s in-place update support", targetInfo.GetNamespace(), targetInfo.GetName())) + continue + } + } + + if targetInfo.GetDeletionTimestamp() != nil { + continue + } + + if opslifecycle.IsDuringOps(r.updateLifecycleAdapter, targetInfo) { + continue + } + + targetCh <- targetToUpdate[i] + } + + // 4. begin target update lifecycle + updating, err = updater.BeginUpdateTarget(ctx, syncContext, targetCh) + if err != nil { + return updating, recordedRequeueAfter, err + } + + // 5. (1) filter out targets not allow to ops now, such as OperationDelaySeconds strategy; (2) update PlaceHolder Targets resourceContext revision + recordedRequeueAfter, err = updater.FilterAllowOpsTargets(ctx, candidates, syncContext.OwnedIds, syncContext, targetCh) + if err != nil { + AddOrUpdateCondition(syncContext.NewStatus, + api.XSetUpdate, err, "UpdateFailed", + fmt.Sprintf("fail to update Context for updating: %s", err)) + return updating, recordedRequeueAfter, err + } else { + AddOrUpdateCondition(syncContext.NewStatus, + api.XSetUpdate, nil, "Updated", "") + } + + // 6. update Target + succCount, err := controllerutils.SlowStartBatch(len(targetCh), controllerutils.SlowStartInitialBatchSize, false, func(_ int, _ error) error { + targetInfo := <-targetCh + logger.V(1).Info("before target update operation", + "target", ObjectKeyString(targetInfo.Object), + "revision.from", targetInfo.CurrentRevision.GetName(), + "revision.to", syncContext.UpdatedRevision.GetName(), + "inPlaceUpdate", targetInfo.InPlaceUpdateSupport, + "onlyMetadataChanged", targetInfo.OnlyMetadataChanged, + ) + + spec := r.xsetController.GetXSetSpec(xsetObject) + isReplaceUpdate := spec.UpdateStrategy.UpdatePolicy == api.XSetReplaceTargetUpdateStrategyType + if targetInfo.IsInReplacing && !isReplaceUpdate { + // a replacing target should be replaced by an updated revision target when encountering upgrade + if err := updateReplaceOriginTarget(ctx, r.Client, r.Recorder, targetInfo, targetInfo.ReplacePairNewTargetInfo); err != nil { + return err + } + } else { + if err := updater.UpgradeTarget(ctx, targetInfo); err != nil { + return err + } + } + + return nil + }) + + updating = updating || succCount > 0 + if err != nil { + AddOrUpdateCondition(syncContext.NewStatus, api.XSetUpdate, err, "UpdateFailed", err.Error()) + return updating, recordedRequeueAfter, err + } else { + AddOrUpdateCondition(syncContext.NewStatus, api.XSetUpdate, nil, "Updated", "") + } + + targetToUpdateSet := sets.String{} + for i := range targetToUpdate { + targetToUpdateSet.Insert(targetToUpdate[i].GetName()) + } + // 7. try to finish all Targets'TargetOpsLifecycle if its update is finished. + succCount, err = controllerutils.SlowStartBatch(len(targetUpdateInfos), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) error { + targetInfo := targetUpdateInfos[i] + + if !targetInfo.IsDuringOps || targetInfo.PlaceHolder || targetInfo.GetDeletionTimestamp() != nil { + return nil + } + + // check Target is during updating, and it is finished or not + finished, msg, err := updater.GetTargetUpdateFinishStatus(ctx, targetInfo) + if err != nil { + return fmt.Errorf("failed to get target %s/%s update finished: %w", targetInfo.GetNamespace(), targetInfo.GetName(), err) + } + + if finished { + if err := updater.FinishUpdateTarget(ctx, targetInfo); err != nil { + return err + } + r.Recorder.Eventf(targetInfo.Object, + corev1.EventTypeNormal, + "UpdateTargetFinished", + "target %s/%s is finished for upgrade to revision %s", + targetInfo.GetNamespace(), targetInfo.GetName(), targetInfo.UpdateRevision.GetName()) + } else { + r.Recorder.Eventf(targetInfo.Object, + corev1.EventTypeNormal, + "WaitingUpdateReady", + "waiting for target %s/%s to update finished: %s", + targetInfo.GetNamespace(), targetInfo.GetName(), msg) + } + + return nil + }) + + return updating || succCount > 0, recordedRequeueAfter, err +} + +func (r *RealSyncControl) CalculateStatus(ctx context.Context, instance api.XSetObject, syncContext *SyncContext) *api.XSetStatus { + newStatus := syncContext.NewStatus + newStatus.ObservedGeneration = instance.GetGeneration() + + var readyReplicas, replicas, updatedReplicas, operatingReplicas, updatedReadyReplicas int32 + + activeTargets := FilterOutActiveTargetWrappers(syncContext.TargetWrappers) + for _, targetWrapper := range activeTargets { + if targetWrapper.GetDeletionTimestamp() != nil { + continue + } + + replicas++ + + isUpdated := false + if isUpdated = IsTargetUpdatedRevision(targetWrapper.Object, syncContext.UpdatedRevision.Name); isUpdated { + updatedReplicas++ + } + + if opslifecycle.IsDuringOps(r.scaleInLifecycleAdapter, targetWrapper) || + opslifecycle.IsDuringOps(r.updateLifecycleAdapter, targetWrapper) { + operatingReplicas++ + } + + if r.xsetController.CheckReady(targetWrapper.Object) { + readyReplicas++ + if isUpdated { + updatedReadyReplicas++ + } + } + } + + newStatus.ReadyReplicas = readyReplicas + newStatus.Replicas = replicas + newStatus.UpdatedReplicas = updatedReplicas + newStatus.OperatingReplicas = operatingReplicas + newStatus.UpdatedReadyReplicas = updatedReadyReplicas + + spec := r.xsetController.GetXSetSpec(instance) + if (spec.Replicas == nil && newStatus.UpdatedReadyReplicas >= 0) || + newStatus.UpdatedReadyReplicas >= *spec.Replicas { + newStatus.CurrentRevision = syncContext.UpdatedRevision.Name + } + + return newStatus +} + +// getAvailableTargetIDs try to extract and re-allocate want available IDs. +func (r *RealSyncControl) getAvailableTargetIDs( + want int, + instance api.XSetObject, + syncContext *SyncContext, +) ([]*appsv1alpha1.ContextDetail, map[int]*appsv1alpha1.ContextDetail, error) { + ownedIDs := syncContext.OwnedIds + currentIDs := syncContext.CurrentIDs + + availableContexts := resourcecontexts.ExtractAvailableContexts(want, ownedIDs, currentIDs) + if len(availableContexts) >= want { + return availableContexts, ownedIDs, nil + } + + diff := want - len(availableContexts) + + var newOwnedIDs map[int]*appsv1alpha1.ContextDetail + var err error + if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + newOwnedIDs, err = resourcecontexts.AllocateID(r.xsetController, r.Client, r.cacheExpectation, instance, syncContext.UpdatedRevision.GetName(), len(ownedIDs)+diff) + return err + }); err != nil { + return nil, ownedIDs, fmt.Errorf("fail to allocate IDs using context when include Targets: %w", err) + } + + return resourcecontexts.ExtractAvailableContexts(want, newOwnedIDs, currentIDs), newOwnedIDs, nil +} + +// reclaimOwnedIDs delete and reclaim unused IDs +func (r *RealSyncControl) reclaimOwnedIDs( + needUpdateContext bool, + xset api.XSetObject, + idToReclaim sets.Int, + ownedIDs map[int]*appsv1alpha1.ContextDetail, + currentIDs sets.Int, +) error { + // TODO stateful case + // 1) only reclaim non-existing Targets' ID. Do not reclaim terminating Targets' ID until these Targets and PVC have been deleted from ETCD + // 2) do not filter out these terminating Targets + for id, contextDetail := range ownedIDs { + if _, exist := currentIDs[id]; exist { + continue + } + if contextDetail.Contains(resourcecontexts.ScaleInContextDataKey, "true") { + idToReclaim.Insert(id) + } + } + + for _, id := range idToReclaim.List() { + needUpdateContext = true + delete(ownedIDs, id) + } + + // TODO clean replace-pair-keys or dirty targetContext + // 1) replace pair target are not exists + // 2) target exists but is not replaceIndicated + + if needUpdateContext { + logger := r.Logger.WithValues(r.xsetGVK.Kind, ObjectKeyString(xset)) + logger.V(1).Info("try to update ResourceContext for XSet when sync") + if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + return resourcecontexts.UpdateToTargetContext(r.xsetController, r.Client, r.cacheExpectation, xset, ownedIDs) + }); err != nil { + return fmt.Errorf("fail to update ResourceContext when reclaiming IDs: %w", err) + } + } + return nil +} + +// FilterOutActiveTargetWrappers filter out non placeholder targets +func FilterOutActiveTargetWrappers(targets []targetWrapper) []*targetWrapper { + var filteredTargetWrappers []*targetWrapper + for i, target := range targets { + if target.PlaceHolder { + continue + } + filteredTargetWrappers = append(filteredTargetWrappers, &targets[i]) + } + return filteredTargetWrappers +} + +func targetDuringReplace(target client.Object) bool { + labels := target.GetLabels() + if labels == nil { + return false + } + _, replaceIndicate := labels[TargetReplaceIndicationLabelKey] + _, replaceOriginTarget := labels[TargetReplacePairNewId] + _, replaceNewTarget := labels[TargetReplacePairOriginName] + return replaceIndicate || replaceOriginTarget || replaceNewTarget +} + +// BatchDelete try to trigger target deletion by to-delete label +func BatchDelete(ctx context.Context, targetControl xcontrol.TargetControl, needDeleteTargets []client.Object) error { + _, err := controllerutils.SlowStartBatch(len(needDeleteTargets), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) error { + return targetControl.DeleteTarget(ctx, needDeleteTargets[i]) + }) + return err +} + +// decideContextRevision decides revision for 3 target create types: (1) just create, (2) upgrade by recreate, (3) delete and recreate +func decideContextRevision(contextDetail *appsv1alpha1.ContextDetail, updatedRevision *appsv1.ControllerRevision, createSucceeded bool) bool { + needUpdateContext := false + if !createSucceeded { + if contextDetail.Contains(resourcecontexts.JustCreateContextDataKey, "true") { + // TODO choose just create targets' revision according to scaleStrategy + contextDetail.Put(resourcecontexts.RevisionContextDataKey, updatedRevision.GetName()) + delete(contextDetail.Data, resourcecontexts.TargetDecorationRevisionKey) + needUpdateContext = true + } else if contextDetail.Contains(resourcecontexts.RecreateUpdateContextDataKey, "true") { + contextDetail.Put(resourcecontexts.RevisionContextDataKey, updatedRevision.GetName()) + delete(contextDetail.Data, resourcecontexts.TargetDecorationRevisionKey) + needUpdateContext = true + } + // if target is delete and recreate, never change revisionKey + } else { + // TODO delete ID if create succeeded + contextDetail.Remove(resourcecontexts.JustCreateContextDataKey) + contextDetail.Remove(resourcecontexts.RecreateUpdateContextDataKey) + needUpdateContext = true + } + return needUpdateContext +} diff --git a/xset/synccontrols/types.go b/xset/synccontrols/types.go new file mode 100644 index 00000000..b0f31e9d --- /dev/null +++ b/xset/synccontrols/types.go @@ -0,0 +1,92 @@ +/* + * Copyright 2024-2025 KusionStack Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package synccontrols + +import ( + "time" + + appsv1 "k8s.io/api/apps/v1" + "k8s.io/apimachinery/pkg/util/sets" + appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "kusionstack.io/kube-utils/xset/api" +) + +type SyncContext struct { + Revisions []*appsv1.ControllerRevision + CurrentRevision *appsv1.ControllerRevision + UpdatedRevision *appsv1.ControllerRevision + ExistingSubResource []client.Object + + FilteredTarget []client.Object + TargetWrappers []targetWrapper + activeTargets []*targetWrapper + replacingMap map[string]*targetWrapper + + CurrentIDs sets.Int + OwnedIds map[int]*appsv1alpha1.ContextDetail + + NewStatus *api.XSetStatus +} + +type targetWrapper struct { + // parameters must be set during creation + client.Object + ID int + ContextDetail *appsv1alpha1.ContextDetail + PlaceHolder bool + + ToDelete bool + ToExclude bool + + IsDuringScaleInOps bool + IsDuringUpdateOps bool +} + +type targetUpdateInfo struct { + *targetWrapper + + UpdatedTarget client.Object + + InPlaceUpdateSupport bool + OnlyMetadataChanged bool + + // indicate if this target has up-to-date revision from its owner, like XSet + IsUpdatedRevision bool + // carry the target's current revision + CurrentRevision *appsv1.ControllerRevision + // carry the desired update revision + UpdateRevision *appsv1.ControllerRevision + + // indicates the TargetOpsLifecycle is started. + IsDuringOps bool + // indicates operate is allowed for TargetOpsLifecycle. + IsAllowOps bool + // requeue after for operationDelaySeconds + RequeueForOperationDelay *time.Duration + + // for replace update + // judge target in replace updating + IsInReplacing bool + + // replace new created target + ReplacePairNewTargetInfo *targetUpdateInfo + + // replace origin target + ReplacePairOriginTargetName string +} diff --git a/xset/synccontrols/x_replace.go b/xset/synccontrols/x_replace.go new file mode 100644 index 00000000..8c5ab468 --- /dev/null +++ b/xset/synccontrols/x_replace.go @@ -0,0 +1,405 @@ +/* +Copyright 2024 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package synccontrols + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "strings" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/tools/record" + appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" + "sigs.k8s.io/controller-runtime/pkg/client" + + controllerutils "kusionstack.io/kube-utils/controller/utils" + "kusionstack.io/kube-utils/xset/api" + "kusionstack.io/kube-utils/xset/opslifecycle" + "kusionstack.io/kube-utils/xset/resourcecontexts" +) + +const ( + ReplaceNewTargetIDContextDataKey = "ReplaceNewTargetID" + ReplaceOriginTargetIDContextDataKey = "ReplaceOriginTargetID" +) + +func (r *RealSyncControl) cleanReplaceTargetLabels(ctx context.Context, needCleanLabelTargets []client.Object, targetsNeedCleanLabels [][]string, ownedIDs map[int]*appsv1alpha1.ContextDetail, currentIDs sets.Int) (bool, sets.Int, error) { + needUpdateContext := false + needDeleteTargetsIDs := sets.Int{} + mapOriginToNewTargetContext := mapReplaceOriginToNewTargetContext(ownedIDs) + mapNewToOriginTargetContext := mapReplaceNewToOriginTargetContext(ownedIDs) + _, err := controllerutils.SlowStartBatch(len(needCleanLabelTargets), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) error { + target := needCleanLabelTargets[i] + needCleanLabels := targetsNeedCleanLabels[i] + var deletePatch []map[string]string + for _, labelKey := range needCleanLabels { + patchOperation := map[string]string{ + "op": "remove", + "path": fmt.Sprintf("/metadata/labels/%s", strings.ReplaceAll(labelKey, "/", "~1")), + } + deletePatch = append(deletePatch, patchOperation) + // replace finished, (1) remove ReplaceNewTargetID, ReplaceOriginTargetID key from IDs, (2) try to delete origin Target's ID + if labelKey == TargetReplacePairOriginName { + needUpdateContext = true + newTargetId, _ := GetInstanceID(target) + if originTargetContext, exist := mapOriginToNewTargetContext[newTargetId]; exist && originTargetContext != nil { + originTargetContext.Remove(ReplaceNewTargetIDContextDataKey) + if _, exist := currentIDs[originTargetContext.ID]; !exist { + needDeleteTargetsIDs.Insert(originTargetContext.ID) + } + } + if contextDetail, exist := ownedIDs[newTargetId]; exist { + contextDetail.Remove(ReplaceOriginTargetIDContextDataKey) + } + } + // replace canceled, (1) remove ReplaceNewTargetID, ReplaceOriginTargetID key from IDs, (2) try to delete new Target's ID + _, replaceIndicate := target.GetLabels()[TargetReplaceIndicationLabelKey] + if !replaceIndicate && labelKey == TargetReplacePairNewId { + needUpdateContext = true + originTargetId, _ := GetInstanceID(target) + if newTargetContext, exist := mapNewToOriginTargetContext[originTargetId]; exist && newTargetContext != nil { + newTargetContext.Remove(ReplaceOriginTargetIDContextDataKey) + if _, exist := currentIDs[newTargetContext.ID]; !exist { + needDeleteTargetsIDs.Insert(newTargetContext.ID) + } + } + if contextDetail, exist := ownedIDs[originTargetId]; exist { + contextDetail.Remove(ReplaceNewTargetIDContextDataKey) + } + } + } + // patch to bytes + patchBytes, err := json.Marshal(deletePatch) + if err != nil { + return err + } + if err = r.xControl.PatchTarget(ctx, target, client.RawPatch(types.JSONPatchType, patchBytes)); err != nil { + return fmt.Errorf("failed to remove replace pair label %s/%s: %w", target.GetNamespace(), target.GetName(), err) + } + return nil + }) + + return needUpdateContext, needDeleteTargetsIDs, err +} + +func (r *RealSyncControl) replaceOriginTargets( + ctx context.Context, + instance api.XSetObject, + syncContext *SyncContext, + needReplaceOriginTargets []client.Object, + ownedIDs map[int]*appsv1alpha1.ContextDetail, + availableContexts []*appsv1alpha1.ContextDetail, +) (int, error) { + mapNewToOriginTargetContext := mapReplaceNewToOriginTargetContext(ownedIDs) + successCount, err := controllerutils.SlowStartBatch(len(needReplaceOriginTargets), controllerutils.SlowStartInitialBatchSize, false, func(i int, _ error) error { + originTarget := needReplaceOriginTargets[i] + originTargetId, _ := GetInstanceID(originTarget) + + replaceRevision := getReplaceRevision(originTarget, syncContext) + + // create target using update revision if replaced by update, otherwise using current revision + newTarget, err := NewTargetFrom(r.xsetController, instance, replaceRevision, originTargetId) + if err != nil { + return err + } + // add instance id and replace pair label + var instanceId string + var newTargetContext *appsv1alpha1.ContextDetail + if contextDetail, exist := mapNewToOriginTargetContext[originTargetId]; exist && contextDetail != nil { + newTargetContext = contextDetail + // reuse targetContext ID if pair-relation exists + instanceId = fmt.Sprintf("%d", newTargetContext.ID) + newTarget.GetLabels()[TargetInstanceIDLabelKey] = instanceId + } else { + if availableContexts[i] == nil { + return fmt.Errorf("cannot found available context for replace new target when replacing origin target %s/%s", originTarget.GetNamespace(), originTarget.GetName()) + } + newTargetContext = availableContexts[i] + // add replace pair-relation to targetContexts for originTarget and newTarget + instanceId = fmt.Sprintf("%d", newTargetContext.ID) + newTarget.GetLabels()[TargetInstanceIDLabelKey] = instanceId + ownedIDs[originTargetId].Put(ReplaceNewTargetIDContextDataKey, instanceId) + ownedIDs[newTargetContext.ID].Put(ReplaceOriginTargetIDContextDataKey, strconv.Itoa(originTargetId)) + ownedIDs[newTargetContext.ID].Remove(resourcecontexts.JustCreateContextDataKey) + } + newTarget.GetLabels()[TargetReplacePairOriginName] = originTarget.GetName() + newTarget.GetLabels()[opslifecycle.TargetCreatingLabel] = strconv.FormatInt(time.Now().UnixNano(), 10) + newTargetContext.Put(resourcecontexts.RevisionContextDataKey, replaceRevision.GetName()) + + if newCreatedTarget, err := r.xControl.CreateTarget(ctx, newTarget); err == nil { + r.Recorder.Eventf(originTarget, + corev1.EventTypeNormal, + "CreatePairTarget", + "succeed to create replace pair Target %s/%s with revision %s by replace", + originTarget.GetNamespace(), + originTarget.GetName(), + replaceRevision.GetName()) + + patch := client.RawPatch(types.MergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:%q}}}`, TargetReplacePairNewId, instanceId))) + if err = r.xControl.PatchTarget(ctx, originTarget, patch); err != nil { + return fmt.Errorf("fail to update origin target %s/%s pair label %s when updating by replaceUpdate: %s", originTarget.GetNamespace(), originTarget.GetName(), newCreatedTarget.GetName(), err.Error()) + } + } else { + r.Recorder.Eventf(originTarget, + corev1.EventTypeNormal, + "ReplaceTarget", + "failed to create replace pair Target %s/%s from revision %s by replace update: %s", + originTarget.GetNamespace(), + originTarget.GetName(), + replaceRevision.GetName(), + err.Error()) + return err + } + return nil + }) + + return successCount, err +} + +func (r *RealSyncControl) dealReplaceTargets(targets []client.Object) (needReplaceTargets, needCleanLabelTargets []client.Object, targetNeedCleanLabels [][]string, needDeleteTargets []client.Object) { + targetInstanceIdMap := make(map[string]client.Object) + targetNameMap := make(map[string]client.Object) + + for _, target := range targets { + targetLabels := target.GetLabels() + + if instanceId, ok := targetLabels[TargetInstanceIDLabelKey]; ok { + targetInstanceIdMap[instanceId] = target + } + targetNameMap[target.GetName()] = target + } + + // deal need replace targets + for _, target := range targets { + targetLabels := target.GetLabels() + + // no replace indication label + if _, exist := targetLabels[TargetReplaceIndicationLabelKey]; !exist { + continue + } + + // origin target is about to scaleIn, skip replace + if opslifecycle.IsDuringOps(r.scaleInLifecycleAdapter, target) { + continue + } + + // target is replace new created target, skip replace + if originTargetName, exist := targetLabels[TargetReplacePairOriginName]; exist { + if _, exist := targetNameMap[originTargetName]; exist { + continue + } + } + + // target already has a new created target for replacement + if newPairTargetId, exist := targetLabels[TargetReplacePairNewId]; exist { + if _, exist := targetInstanceIdMap[newPairTargetId]; exist { + continue + } + } + + needReplaceTargets = append(needReplaceTargets, target) + } + + for _, target := range targets { + targetLabels := target.GetLabels() + _, replaceByUpdate := targetLabels[TargetReplaceByReplaceUpdateLabelKey] + var needCleanLabels []string + + // target is replace new created target, skip replace + if originTargetName, exist := targetLabels[TargetReplacePairOriginName]; exist { + // replace pair origin target is not exist, clean label. + if originTarget, exist := targetNameMap[originTargetName]; !exist { + needCleanLabels = append(needCleanLabels, TargetReplacePairOriginName) + } else if originTarget.GetLabels()[TargetReplaceIndicationLabelKey] == "" { + // replace canceled, delete replace new target if new target is not service available + if _, exist := targetLabels[opslifecycle.TargetServiceAvailableLabel]; !exist { + needDeleteTargets = append(needDeleteTargets, target) + } + } else if !replaceByUpdate { + // not replace update, delete origin target when new created target is service available + if _, serviceAvailable := targetLabels[opslifecycle.TargetServiceAvailableLabel]; serviceAvailable { + needDeleteTargets = append(needDeleteTargets, originTarget) + } + } + } + + if newPairTargetId, exist := targetLabels[TargetReplacePairNewId]; exist { + if _, exist := targetInstanceIdMap[newPairTargetId]; !exist { + needCleanLabels = append(needCleanLabels, TargetReplacePairNewId) + } + } + + if len(needCleanLabels) > 0 { + needCleanLabelTargets = append(needCleanLabelTargets, target) + targetNeedCleanLabels = append(targetNeedCleanLabels, needCleanLabels) + } + } + return needReplaceTargets, needCleanLabelTargets, targetNeedCleanLabels, needDeleteTargets +} + +func updateReplaceOriginTarget( + ctx context.Context, + c client.Client, + recorder record.EventRecorder, + originTargetUpdateInfo, newTargetUpdateInfo *targetUpdateInfo, +) error { + originTarget := originTargetUpdateInfo.Object + + // 1. delete the new target if not updated + if newTargetUpdateInfo != nil { + newTarget := newTargetUpdateInfo.Object + _, deletionIndicate := newTarget.GetLabels()[TargetDeletionIndicationLabelKey] + currentRevision, exist := newTarget.GetLabels()[appsv1.ControllerRevisionHashLabelKey] + if exist && currentRevision != originTargetUpdateInfo.UpdateRevision.GetName() && !deletionIndicate { + patch := client.RawPatch(types.MergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:"%d"}}}`, TargetDeletionIndicationLabelKey, time.Now().UnixNano()))) + if patchErr := c.Patch(ctx, newTarget, patch); patchErr != nil { + err := fmt.Errorf("failed to delete replace pair new target %s/%s %s", + newTarget.GetNamespace(), newTarget.GetName(), patchErr.Error()) + return err + } + recorder.Eventf(originTarget, + corev1.EventTypeNormal, + "DeleteOldNewTarget", + "succeed to delete replace new Target %s/%s by label to-replace", + originTarget.GetNamespace(), + originTarget.GetName(), + ) + } + } + + // 2. replace the origin target with updated target + _, replaceIndicate := originTarget.GetLabels()[TargetReplaceIndicationLabelKey] + _, replaceByUpdate := originTarget.GetLabels()[TargetReplaceByReplaceUpdateLabelKey] + if !replaceIndicate || !replaceByUpdate { + now := time.Now().UnixNano() + patch := client.RawPatch(types.MergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:"%v", %q: "%v"}}}`, TargetReplaceIndicationLabelKey, now, TargetReplaceByReplaceUpdateLabelKey, originTargetUpdateInfo.UpdateRevision.GetName()))) + if err := c.Patch(ctx, originTarget, patch); err != nil { + return fmt.Errorf("fail to label origin target %s/%s with replace indicate label by replaceUpdate: %s", originTarget.GetNamespace(), originTarget.GetName(), err.Error()) + } + recorder.Eventf(originTarget, + corev1.EventTypeNormal, + "UpdateOriginTarget", + "succeed to update Target %s/%s by label to-replace", + originTarget.GetNamespace(), + originTarget.GetName(), + ) + } + + return nil +} + +// getReplaceRevision finds replaceNewTarget's revision from originTarget +func getReplaceRevision(originTarget client.Object, syncContext *SyncContext) *appsv1.ControllerRevision { + // replace update, first find revision from label, if revision not found, just replace with updated revision + if updateRevisionName, exist := originTarget.GetLabels()[TargetReplaceByReplaceUpdateLabelKey]; exist { + for _, rv := range syncContext.Revisions { + if updateRevisionName == rv.GetName() { + return rv + } + } + return syncContext.UpdatedRevision + } + + // replace by to-replace label, just replace with current revision + targetCurrentRevisionName, exist := originTarget.GetLabels()[appsv1.ControllerRevisionHashLabelKey] + if !exist { + return syncContext.CurrentRevision + } + + for _, revision := range syncContext.Revisions { + if revision.GetName() == targetCurrentRevisionName { + return revision + } + } + + return syncContext.CurrentRevision +} + +// classify the pair relationship for Target replacement. +func classifyTargetReplacingMapping(targetWrappers []*targetWrapper) map[string]*targetWrapper { + targetNameMap := make(map[string]*targetWrapper) + targetIdMap := make(map[string]*targetWrapper) + for _, targetWrapper := range targetWrappers { + targetNameMap[targetWrapper.GetName()] = targetWrapper + targetIdMap[strconv.Itoa(targetWrapper.ID)] = targetWrapper + } + + // old target name => new target wrapper + replaceTargetMapping := make(map[string]*targetWrapper) + for _, targetWrapper := range targetWrappers { + if targetWrapper.Object == nil { + continue + } + name := targetWrapper.GetName() + if replacePairNewIdStr, exist := targetWrapper.GetLabels()[TargetReplacePairNewId]; exist { + if pairNewTarget, exist := targetIdMap[replacePairNewIdStr]; exist { + replaceTargetMapping[name] = pairNewTarget + // if one of pair targets is to Exclude, both targets should not scaleIn + targetWrapper.ToExclude = targetWrapper.ToExclude || pairNewTarget.ToExclude + continue + } + } else if replaceOriginStr, exist := targetWrapper.GetLabels()[TargetReplacePairOriginName]; exist { + if originTarget, exist := targetNameMap[replaceOriginStr]; exist { + if originTarget.GetLabels()[TargetReplacePairNewId] == strconv.Itoa(targetWrapper.ID) { + continue + } + } + } + + // non paired target, just put it in the map + replaceTargetMapping[name] = nil + } + return replaceTargetMapping +} + +func mapReplaceNewToOriginTargetContext(ownedIDs map[int]*appsv1alpha1.ContextDetail) map[int]*appsv1alpha1.ContextDetail { + mapNewToOriginTargetContext := make(map[int]*appsv1alpha1.ContextDetail) + for id, contextDetail := range ownedIDs { + if val, exist := contextDetail.Data[ReplaceNewTargetIDContextDataKey]; exist { + newTargetId, _ := strconv.ParseInt(val, 10, 32) + newTargetContextDetail, exist := ownedIDs[int(newTargetId)] + if exist && newTargetContextDetail.Data[ReplaceOriginTargetIDContextDataKey] == strconv.Itoa(id) { + mapNewToOriginTargetContext[id] = newTargetContextDetail + } else { + mapNewToOriginTargetContext[id] = nil + } + } + } + return mapNewToOriginTargetContext +} + +func mapReplaceOriginToNewTargetContext(ownedIDs map[int]*appsv1alpha1.ContextDetail) map[int]*appsv1alpha1.ContextDetail { + mapOriginToNewTargetContext := make(map[int]*appsv1alpha1.ContextDetail) + for id, contextDetail := range ownedIDs { + if val, exist := contextDetail.Data[ReplaceOriginTargetIDContextDataKey]; exist { + originTargetId, _ := strconv.ParseInt(val, 10, 32) + originTargetContextDetail, exist := ownedIDs[int(originTargetId)] + if exist && originTargetContextDetail.Data[ReplaceNewTargetIDContextDataKey] == strconv.Itoa(id) { + mapOriginToNewTargetContext[id] = originTargetContextDetail + } else { + mapOriginToNewTargetContext[id] = nil + } + } + } + return mapOriginToNewTargetContext +} diff --git a/xset/synccontrols/x_scale.go b/xset/synccontrols/x_scale.go new file mode 100644 index 00000000..35a5d854 --- /dev/null +++ b/xset/synccontrols/x_scale.go @@ -0,0 +1,174 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package synccontrols + +import ( + "context" + "sort" + "strconv" + + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/client-go/util/retry" + appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" + + controllerutils "kusionstack.io/kube-utils/controller/utils" + "kusionstack.io/kube-utils/xset/api" + "kusionstack.io/kube-utils/xset/opslifecycle" +) + +// getTargetsToDelete finds number of diff targets from filteredTargets to do scaleIn +func (r *RealSyncControl) getTargetsToDelete(filteredTargets []*targetWrapper, replaceMapping map[string]*targetWrapper, diff int) []*targetWrapper { + var countedTargets []*targetWrapper + for _, target := range filteredTargets { + if _, exist := replaceMapping[target.GetName()]; exist { + countedTargets = append(countedTargets, target) + } + } + + // 1. select targets to delete in first round according to diff + sort.Sort(newActiveTargetsForDeletion(countedTargets)) + if diff > len(countedTargets) { + diff = len(countedTargets) + } + + // 2. select targets to delete in second round according to replace, delete, exclude + var needDeleteTargets []*targetWrapper + for _, target := range countedTargets[:diff] { + // don't scaleIn exclude target and its newTarget (if exist) + if target.ToExclude { + continue + } + + if replacePairTarget, exist := replaceMapping[target.GetName()]; exist && replacePairTarget != nil { + // don't selective scaleIn newTarget (and its originTarget) until replace finished + if replacePairTarget.ToDelete && !target.ToDelete { + continue + } + // when scaleIn origin Target, newTarget should be deleted if not service available + if _, serviceAvailable := replacePairTarget.GetLabels()[opslifecycle.TargetServiceAvailableLabel]; !serviceAvailable { + needDeleteTargets = append(needDeleteTargets, replacePairTarget) + } + } + needDeleteTargets = append(needDeleteTargets, target) + } + + return needDeleteTargets +} + +type ActiveTargetsForDeletion struct { + targets []*targetWrapper +} + +func newActiveTargetsForDeletion(targets []*targetWrapper) *ActiveTargetsForDeletion { + return &ActiveTargetsForDeletion{ + targets: targets, + } +} + +func (s *ActiveTargetsForDeletion) Len() int { return len(s.targets) } +func (s *ActiveTargetsForDeletion) Swap(i, j int) { + s.targets[i], s.targets[j] = s.targets[j], s.targets[i] +} + +// Less sort deletion order by: targetToDelete > targetToExclude > duringScaleIn > others +func (s *ActiveTargetsForDeletion) Less(i, j int) bool { + l, r := s.targets[i], s.targets[j] + + if l.ToDelete != r.ToDelete { + return l.ToDelete + } + + if l.ToExclude != r.ToExclude { + return l.ToExclude + } + + // targets which are during scaleInOps should be deleted before those not during + if l.IsDuringScaleInOps != r.IsDuringScaleInOps { + return l.IsDuringScaleInOps + } + + // TODO consider service available timestamps + + lCreationTime := l.Object.GetCreationTimestamp().Time + rCreationTime := r.Object.GetCreationTimestamp().Time + return lCreationTime.Before(rCreationTime) +} + +// doIncludeExcludeTargets do real include and exclude for targets which are allowed to in/exclude +func (r *RealSyncControl) doIncludeExcludeTargets(ctx context.Context, xset api.XSetObject, excludeTargets, includeTargets []string, availableContexts []*appsv1alpha1.ContextDetail) error { + var excludeErrs, includeErrs []error + _, _ = controllerutils.SlowStartBatch(len(excludeTargets), controllerutils.SlowStartInitialBatchSize, false, func(idx int, _ error) (err error) { + defer func() { excludeErrs = append(excludeErrs, err) }() + return r.excludeTarget(ctx, xset, excludeTargets[idx]) + }) + _, _ = controllerutils.SlowStartBatch(len(includeTargets), controllerutils.SlowStartInitialBatchSize, false, func(idx int, _ error) (err error) { + defer func() { includeErrs = append(includeErrs, err) }() + return r.includeTarget(ctx, xset, includeTargets[idx], strconv.Itoa(availableContexts[idx].ID)) + }) + return controllerutils.AggregateErrors(append(includeErrs, excludeErrs...)) +} + +// excludeTarget try to exclude a target from xset +func (r *RealSyncControl) excludeTarget(ctx context.Context, xsetObject api.XSetObject, targetName string) error { + target := r.xsetController.EmptyXObject() + if err := r.Client.Get(ctx, types.NamespacedName{Namespace: xsetObject.GetNamespace(), Name: targetName}, target); err != nil { + return err + } + + target.GetLabels()[TargetOrphanedIndicateLabelKey] = "true" + return r.xControl.OrphanTarget(xsetObject, target) +} + +// includeTarget try to include a target into xset +func (r *RealSyncControl) includeTarget(ctx context.Context, xsetObject api.XSetObject, targetName, instanceId string) error { + target := r.xsetController.EmptyXObject() + if err := r.Client.Get(ctx, types.NamespacedName{Namespace: xsetObject.GetNamespace(), Name: targetName}, target); err != nil { + return err + } + + target.GetLabels()[TargetInstanceIDLabelKey] = instanceId + delete(target.GetLabels(), TargetOrphanedIndicateLabelKey) + return r.xControl.AdoptTarget(xsetObject, target) +} + +// reclaimScaleStrategy updates targetToDelete, targetToExclude, targetToInclude in scaleStrategy +func (r *RealSyncControl) reclaimScaleStrategy(ctx context.Context, deletedTargets, excludedTargets, includedTargets sets.String, xsetObject api.XSetObject) error { + xspec := r.xsetController.GetXSetSpec(xsetObject) + // reclaim TargetToDelete + toDeleteTargets := sets.NewString(xspec.ScaleStrategy.TargetToDelete...) + notDeletedTargets := toDeleteTargets.Delete(deletedTargets.List()...) + xspec.ScaleStrategy.TargetToDelete = notDeletedTargets.List() + // reclaim TargetToExclude + toExcludeTargets := sets.NewString(xspec.ScaleStrategy.TargetToExclude...) + notExcludeTargets := toExcludeTargets.Delete(excludedTargets.List()...) + xspec.ScaleStrategy.TargetToExclude = notExcludeTargets.List() + // reclaim TargetToInclude + toIncludeTargetNames := sets.NewString(xspec.ScaleStrategy.TargetToInclude...) + notIncludeTargets := toIncludeTargetNames.Delete(includedTargets.List()...) + xspec.ScaleStrategy.TargetToInclude = notIncludeTargets.List() + if err := r.xsetController.UpdateScaleStrategy(xsetObject, &xspec.ScaleStrategy); err != nil { + return err + } + // update xsetObject.spec.scaleStrategy + if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + return r.Client.Update(ctx, xsetObject) + }); err != nil { + return err + } + return r.cacheExpectation.ExpectUpdation(r.xsetGVK, xsetObject.GetNamespace(), xsetObject.GetName(), xsetObject.GetResourceVersion()) +} diff --git a/xset/synccontrols/x_update.go b/xset/synccontrols/x_update.go new file mode 100644 index 00000000..8256de60 --- /dev/null +++ b/xset/synccontrols/x_update.go @@ -0,0 +1,691 @@ +/* +Copyright 2023 The KusionStack Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package synccontrols + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sort" + "time" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/retry" + "k8s.io/utils/ptr" + appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "kusionstack.io/kube-utils/controller/expectations" + "kusionstack.io/kube-utils/controller/merge" + controllerutils "kusionstack.io/kube-utils/controller/utils" + "kusionstack.io/kube-utils/xset/api" + "kusionstack.io/kube-utils/xset/opslifecycle" + "kusionstack.io/kube-utils/xset/resourcecontexts" + "kusionstack.io/kube-utils/xset/xcontrol" +) + +const UnknownRevision = "__unknownRevision__" + +func (r *RealSyncControl) attachTargetUpdateInfo(xsetObject api.XSetObject, syncContext *SyncContext) []*targetUpdateInfo { + activeTargets := FilterOutActiveTargetWrappers(syncContext.TargetWrappers) + targetUpdateInfoList := make([]*targetUpdateInfo, len(activeTargets)) + + for i, target := range activeTargets { + updateInfo := &targetUpdateInfo{ + targetWrapper: &syncContext.TargetWrappers[i], + InPlaceUpdateSupport: true, + } + + updateInfo.UpdateRevision = syncContext.UpdatedRevision + // decide this target current revision, or nil if not indicated + if target.GetLabels() != nil { + currentRevisionName, exist := target.GetLabels()[appsv1.ControllerRevisionHashLabelKey] + if exist { + if currentRevisionName == syncContext.UpdatedRevision.GetName() { + updateInfo.IsUpdatedRevision = true + updateInfo.CurrentRevision = syncContext.UpdatedRevision + } else { + updateInfo.IsUpdatedRevision = false + for _, rv := range syncContext.Revisions { + if currentRevisionName == rv.GetName() { + updateInfo.CurrentRevision = rv + } + } + } + } + } + + // default CurrentRevision is an empty revision + if updateInfo.CurrentRevision == nil { + updateInfo.CurrentRevision = &appsv1.ControllerRevision{ + ObjectMeta: metav1.ObjectMeta{ + Name: UnknownRevision, + }, + } + r.Recorder.Eventf(target.Object, + corev1.EventTypeWarning, + "TargetCurrentRevisionNotFound", + "target is going to be updated by recreate because: (1) controller-revision-hash label not found, or (2) not found in history revisions") + } + + spec := r.xsetController.GetXSetSpec(xsetObject) + // decide whether the TargetOpsLifecycle is during ops or not + updateInfo.IsDuringOps = target.IsDuringUpdateOps + updateInfo.RequeueForOperationDelay, updateInfo.IsAllowOps = opslifecycle.AllowOps(r.updateLifecycleAdapter, RealValue(spec.UpdateStrategy.OperationDelaySeconds), target) + + targetUpdateInfoList[i] = updateInfo + } + + // attach replace info + targetUpdateInfoMap := make(map[string]*targetUpdateInfo) + for _, targetUpdateInfo := range targetUpdateInfoList { + targetUpdateInfoMap[targetUpdateInfo.GetName()] = targetUpdateInfo + } + for originTargetName, replacePairNewTarget := range syncContext.replacingMap { + originTargetInfo := targetUpdateInfoMap[originTargetName] + if replacePairNewTarget != nil { + originTargetInfo.IsInReplacing = true + // replace origin target not go through lifecycle, mark during ops manual + originTargetInfo.IsDuringOps = true + originTargetInfo.IsAllowOps = true + ReplacePairNewTargetInfo := targetUpdateInfoMap[replacePairNewTarget.GetName()] + ReplacePairNewTargetInfo.IsInReplacing = true + ReplacePairNewTargetInfo.ReplacePairOriginTargetName = originTargetName + originTargetInfo.ReplacePairNewTargetInfo = ReplacePairNewTargetInfo + } else { + _, replaceIndicated := originTargetInfo.GetLabels()[TargetReplaceIndicationLabelKey] + _, replaceByReplaceUpdate := originTargetInfo.GetLabels()[TargetReplaceByReplaceUpdateLabelKey] + if replaceIndicated && replaceByReplaceUpdate { + originTargetInfo.IsInReplacing = true + originTargetInfo.IsDuringOps = true + originTargetInfo.IsAllowOps = true + } + } + } + + // join PlaceHolder targets in updating + for _, target := range syncContext.TargetWrappers { + if !target.PlaceHolder { + continue + } + updateInfo := &targetUpdateInfo{ + targetWrapper: &target, + InPlaceUpdateSupport: true, + UpdateRevision: syncContext.UpdatedRevision, + } + if revision, exist := target.ContextDetail.Data[resourcecontexts.RevisionContextDataKey]; exist && + revision == syncContext.UpdatedRevision.GetName() { + updateInfo.IsUpdatedRevision = true + } + targetUpdateInfoList = append(targetUpdateInfoList, updateInfo) + } + + return targetUpdateInfoList +} + +func filterOutPlaceHolderUpdateInfos(targets []*targetUpdateInfo) []*targetUpdateInfo { + var filteredTargetUpdateInfos []*targetUpdateInfo + for _, target := range targets { + if target.PlaceHolder { + continue + } + filteredTargetUpdateInfos = append(filteredTargetUpdateInfos, target) + } + return filteredTargetUpdateInfos +} + +func decideTargetToUpdate(xsetController api.XSetController, xset api.XSetObject, targetInfos []*targetUpdateInfo) []*targetUpdateInfo { + spec := xsetController.GetXSetSpec(xset) + if spec.UpdateStrategy.RollingUpdate != nil && spec.UpdateStrategy.RollingUpdate.ByLabel != nil { + activeTargetInfos := filterOutPlaceHolderUpdateInfos(targetInfos) + return decideTargetToUpdateByLabel(xset, activeTargetInfos) + } + + return decideTargetToUpdateByPartition(xsetController, xset, targetInfos) +} + +func decideTargetToUpdateByLabel(_ api.XSetObject, targetInfos []*targetUpdateInfo) (targetToUpdate []*targetUpdateInfo) { + for i := range targetInfos { + if _, exist := targetInfos[i].GetLabels()[XSetUpdateIndicateLabelKey]; exist { + // filter target which is in replace update and is the new created target + if targetInfos[i].IsInReplacing && targetInfos[i].ReplacePairOriginTargetName != "" { + continue + } + targetToUpdate = append(targetToUpdate, targetInfos[i]) + continue + } + + // already in replace update. + if targetInfos[i].IsInReplacing && targetInfos[i].ReplacePairNewTargetInfo != nil { + targetToUpdate = append(targetToUpdate, targetInfos[i]) + continue + } + } + return targetToUpdate +} + +func decideTargetToUpdateByPartition(xsetController api.XSetController, xset api.XSetObject, targetInfos []*targetUpdateInfo) []*targetUpdateInfo { + spec := xsetController.GetXSetSpec(xset) + replicas := ptr.Deref(spec.Replicas, 0) + partition := int32(0) + + if spec.UpdateStrategy.RollingUpdate != nil && spec.UpdateStrategy.RollingUpdate.ByPartition != nil { + partition = ptr.Deref(spec.UpdateStrategy.RollingUpdate.ByPartition.Partition, 0) + } + + filteredTargetInfos := getTargetsUpdateTargets(targetInfos) + // update all or not update any replicas + if partition == 0 { + return filteredTargetInfos + } + if partition >= replicas { + return nil + } + + // partial update replicas + ordered := newOrderedTargetUpdateInfos(filteredTargetInfos, xsetController.CheckReady) + sort.Sort(ordered) + targetToUpdate := ordered.targets[:replicas-partition] + return targetToUpdate +} + +// when sort targets to choose update, only sort (1) replace origin targets, (2) non-exclude targets +func getTargetsUpdateTargets(targetInfos []*targetUpdateInfo) (filteredTargetInfos []*targetUpdateInfo) { + for _, targetInfo := range targetInfos { + if targetInfo.IsInReplacing && targetInfo.ReplacePairOriginTargetName != "" { + continue + } + + if targetInfo.PlaceHolder { + _, isReplaceNewTarget := targetInfo.ContextDetail.Data[ReplaceOriginTargetIDContextDataKey] + _, isReplaceOriginTarget := targetInfo.ContextDetail.Data[ReplaceNewTargetIDContextDataKey] + if isReplaceNewTarget || isReplaceOriginTarget { + continue + } + } + + filteredTargetInfos = append(filteredTargetInfos, targetInfo) + } + return filteredTargetInfos +} + +func newOrderedTargetUpdateInfos(targetInfos []*targetUpdateInfo, checkReadyFunc func(object client.Object) bool) *orderByDefault { + return &orderByDefault{ + targets: targetInfos, + checkReadyFunc: checkReadyFunc, + } +} + +type orderByDefault struct { + targets []*targetUpdateInfo + checkReadyFunc func(object client.Object) bool +} + +func (o *orderByDefault) Len() int { + return len(o.targets) +} + +func (o *orderByDefault) Swap(i, j int) { o.targets[i], o.targets[j] = o.targets[j], o.targets[i] } + +func (o *orderByDefault) Less(i, j int) bool { + l, r := o.targets[i], o.targets[j] + if l.IsUpdatedRevision != r.IsUpdatedRevision { + return l.IsUpdatedRevision + } + + if l.PlaceHolder != r.PlaceHolder { + return r.PlaceHolder + } + + if l.PlaceHolder && r.PlaceHolder { + return true + } + + if l.IsDuringOps != r.IsDuringOps { + return l.IsDuringOps + } + + lReady, rReady := o.checkReadyFunc(l.Object), o.checkReadyFunc(r.Object) + if lReady != rReady { + return lReady + } + + lCreationTime := l.Object.GetCreationTimestamp().Time + rCreationTime := r.Object.GetCreationTimestamp().Time + return lCreationTime.Before(rCreationTime) +} + +type UpdateConfig struct { + xsetController api.XSetController + client client.Client + targetControl xcontrol.TargetControl + recorder record.EventRecorder + + scaleInLifecycleAdapter api.LifecycleAdapter + updateLifecycleAdapter api.LifecycleAdapter + + cacheExpectation *expectations.CacheExpectation + targetGVK schema.GroupVersionKind +} + +type TargetUpdater interface { + Setup(config *UpdateConfig, xset api.XSetObject) + FulfillTargetUpdatedInfo(ctx context.Context, revision *appsv1.ControllerRevision, targetUpdateInfo *targetUpdateInfo) error + BeginUpdateTarget(ctx context.Context, syncContext *SyncContext, targetCh chan *targetUpdateInfo) (bool, error) + FilterAllowOpsTargets(ctx context.Context, targetToUpdate []*targetUpdateInfo, ownedIDs map[int]*appsv1alpha1.ContextDetail, syncContext *SyncContext, targetCh chan *targetUpdateInfo) (*time.Duration, error) + UpgradeTarget(ctx context.Context, targetInfo *targetUpdateInfo) error + GetTargetUpdateFinishStatus(ctx context.Context, targetUpdateInfo *targetUpdateInfo) (bool, string, error) + FinishUpdateTarget(ctx context.Context, targetInfo *targetUpdateInfo) error +} + +type GenericTargetUpdater struct { + OwnerObject api.XSetObject + + *UpdateConfig +} + +func (u *GenericTargetUpdater) Setup(config *UpdateConfig, xset api.XSetObject) { + u.UpdateConfig = config + u.OwnerObject = xset +} + +func (u *GenericTargetUpdater) BeginUpdateTarget(_ context.Context, syncContext *SyncContext, targetCh chan *targetUpdateInfo) (bool, error) { + succCount, err := controllerutils.SlowStartBatch(len(targetCh), controllerutils.SlowStartInitialBatchSize, false, func(int, error) error { + targetInfo := <-targetCh + u.recorder.Eventf(targetInfo.Object, corev1.EventTypeNormal, "TargetUpdateLifecycle", "try to begin TargetOpsLifecycle for updating Target of XSet") + + if updated, err := opslifecycle.BeginWithCleaningOld(u.client, u.updateLifecycleAdapter, targetInfo.Object, func(obj client.Object) (bool, error) { + if !targetInfo.OnlyMetadataChanged && !targetInfo.InPlaceUpdateSupport { + return opslifecycle.WhenBeginDelete(obj) + } + return false, nil + }); err != nil { + return fmt.Errorf("fail to begin TargetOpsLifecycle for updating Target %s/%s: %s", targetInfo.GetNamespace(), targetInfo.GetName(), err.Error()) + } else if updated { + // add an expectation for this target update, before next reconciling + if err := u.cacheExpectation.ExpectUpdation(u.targetGVK, targetInfo.GetNamespace(), targetInfo.GetName(), targetInfo.GetResourceVersion()); err != nil { + return err + } + } + + return nil + }) + + updating := succCount > 0 + if err != nil { + AddOrUpdateCondition(syncContext.NewStatus, api.XSetUpdate, err, "UpdateFailed", err.Error()) + return updating, err + } else { + AddOrUpdateCondition(syncContext.NewStatus, api.XSetUpdate, nil, "Updated", "") + } + return updating, nil +} + +func (u *GenericTargetUpdater) FilterAllowOpsTargets(_ context.Context, candidates []*targetUpdateInfo, ownedIDs map[int]*appsv1alpha1.ContextDetail, _ *SyncContext, targetCh chan *targetUpdateInfo) (*time.Duration, error) { + var recordedRequeueAfter *time.Duration + needUpdateContext := false + for i := range candidates { + targetInfo := candidates[i] + + if !targetInfo.PlaceHolder { + if !targetInfo.IsAllowOps { + continue + } + if targetInfo.RequeueForOperationDelay != nil { + u.recorder.Eventf(targetInfo, corev1.EventTypeNormal, "TargetUpdateLifecycle", "delay Target update for %f seconds", targetInfo.RequeueForOperationDelay.Seconds()) + if recordedRequeueAfter == nil || *targetInfo.RequeueForOperationDelay < *recordedRequeueAfter { + recordedRequeueAfter = targetInfo.RequeueForOperationDelay + } + continue + } + } + + targetInfo.IsAllowOps = true + + if targetInfo.IsUpdatedRevision { + continue + } + + if _, exist := ownedIDs[targetInfo.ID]; !exist { + u.recorder.Eventf(u.OwnerObject, corev1.EventTypeWarning, "TargetBeforeUpdate", "target %s/%s is not allowed to update because cannot find context id %s in resourceContext", targetInfo.GetNamespace(), targetInfo.GetName(), targetInfo.GetLabels()[TargetInstanceIDLabelKey]) + continue + } + + if !ownedIDs[targetInfo.ID].Contains(resourcecontexts.RevisionContextDataKey, targetInfo.UpdateRevision.GetName()) { + needUpdateContext = true + ownedIDs[targetInfo.ID].Put(resourcecontexts.RevisionContextDataKey, targetInfo.UpdateRevision.GetName()) + } + + spec := u.xsetController.GetXSetSpec(u.OwnerObject) + + // mark targetContext "TargetRecreateUpgrade" if upgrade by recreate + isRecreateUpdatePolicy := spec.UpdateStrategy.UpdatePolicy == api.XSetRecreateTargetUpdateStrategyType + if (!targetInfo.OnlyMetadataChanged && !targetInfo.InPlaceUpdateSupport) || isRecreateUpdatePolicy { + ownedIDs[targetInfo.ID].Put(resourcecontexts.RecreateUpdateContextDataKey, "true") + } + + if targetInfo.PlaceHolder { + continue + } + + // if Target has not been updated, update it. + targetCh <- candidates[i] + } + // mark Target to use updated revision before updating it. + if needUpdateContext { + u.recorder.Eventf(u.OwnerObject, corev1.EventTypeNormal, "UpdateToTargetContext", "try to update ResourceContext for XSet") + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + return resourcecontexts.UpdateToTargetContext(u.xsetController, u.client, u.cacheExpectation, u.OwnerObject, ownedIDs) + }) + return recordedRequeueAfter, err + } + return recordedRequeueAfter, nil +} + +func (u *GenericTargetUpdater) FinishUpdateTarget(_ context.Context, targetInfo *targetUpdateInfo) error { + if updated, err := opslifecycle.Finish(u.client, u.updateLifecycleAdapter, targetInfo.Object); err != nil { + return fmt.Errorf("failed to finish TargetOpsLifecycle for updating Target %s/%s: %s", targetInfo.GetNamespace(), targetInfo.GetName(), err.Error()) + } else if updated { + // add an expectation for this target update, before next reconciling + if err := u.cacheExpectation.ExpectUpdation(u.targetGVK, targetInfo.GetNamespace(), targetInfo.GetName(), targetInfo.GetResourceVersion()); err != nil { + return err + } + u.recorder.Eventf(targetInfo.Object, + corev1.EventTypeNormal, + "UpdateReady", "target %s/%s update finished", targetInfo.GetNamespace(), targetInfo.GetName()) + } + return nil +} + +// Support users to define inPlaceOnlyTargetUpdater and register through RegisterInPlaceOnlyUpdater +var inPlaceOnlyTargetUpdater TargetUpdater + +func RegisterInPlaceOnlyUpdater(targetUpdater TargetUpdater) { + inPlaceOnlyTargetUpdater = targetUpdater +} + +func (r *RealSyncControl) newTargetUpdater(xset api.XSetObject) TargetUpdater { + spec := r.xsetController.GetXSetSpec(xset) + var targetUpdater TargetUpdater + switch spec.UpdateStrategy.UpdatePolicy { + case api.XSetRecreateTargetUpdateStrategyType: + targetUpdater = &recreateTargetUpdater{} + case api.XSetInPlaceOnlyTargetUpdateStrategyType: + if inPlaceOnlyTargetUpdater != nil { + targetUpdater = inPlaceOnlyTargetUpdater + } else { + // In case of using native K8s, Target is only allowed to update with container image, so InPlaceOnly policy is + // implemented with InPlaceIfPossible policy as default for compatibility. + targetUpdater = &inPlaceIfPossibleUpdater{} + } + case api.XSetReplaceTargetUpdateStrategyType: + targetUpdater = &replaceUpdateTargetUpdater{} + default: + targetUpdater = &inPlaceIfPossibleUpdater{} + } + targetUpdater.Setup(r.updateConfig, xset) + return targetUpdater +} + +type TargetStatus struct { + ContainerStates map[string]*ContainerStatus `json:"containerStates,omitempty"` +} + +type ContainerStatus struct { + LatestImage string `json:"latestImage,omitempty"` + LastImageID string `json:"lastImageID,omitempty"` +} + +type inPlaceIfPossibleUpdater struct { + GenericTargetUpdater +} + +func (u *inPlaceIfPossibleUpdater) FulfillTargetUpdatedInfo(_ context.Context, revision *appsv1.ControllerRevision, targetUpdateInfo *targetUpdateInfo) error { + // 1. build target from current and updated revision + // TODO: use cache + currentTarget, err := NewTargetFrom(u.xsetController, u.OwnerObject, targetUpdateInfo.CurrentRevision, targetUpdateInfo.ID) + if err != nil { + return fmt.Errorf("fail to build Target from current revision %s: %v", targetUpdateInfo.CurrentRevision.GetName(), err.Error()) + } + + // TODO: use cache + + UpdatedTarget, err := NewTargetFrom(u.xsetController, u.OwnerObject, targetUpdateInfo.UpdateRevision, targetUpdateInfo.ID) + if err != nil { + return fmt.Errorf("fail to build Target from updated revision %s: %v", targetUpdateInfo.UpdateRevision.GetName(), err.Error()) + } + + newUpdatedTarget := targetUpdateInfo.targetWrapper.Object.DeepCopyObject().(client.Object) + if err = merge.ThreeWayMergeToTarget(currentTarget, UpdatedTarget, newUpdatedTarget, u.xsetController.EmptyXObject()); err != nil { + return fmt.Errorf("fail to patch Target %s/%s: %v", targetUpdateInfo.GetNamespace(), targetUpdateInfo.GetName(), err.Error()) + } + targetUpdateInfo.UpdatedTarget = newUpdatedTarget + + return nil +} + +func (u *inPlaceIfPossibleUpdater) UpgradeTarget(ctx context.Context, targetInfo *targetUpdateInfo) error { + if targetInfo.OnlyMetadataChanged || targetInfo.InPlaceUpdateSupport { + // if target template changes only include metadata or support in-place update, just apply these changes to target directly + if err := u.targetControl.UpdateTarget(ctx, targetInfo.UpdatedTarget); err != nil { + return fmt.Errorf("fail to update Target %s/%s when updating by in-place: %s", targetInfo.GetNamespace(), targetInfo.GetName(), err.Error()) + } else { + targetInfo.Object = targetInfo.UpdatedTarget + u.recorder.Eventf(targetInfo.Object, + corev1.EventTypeNormal, + "UpdateTarget", + "succeed to update Target %s/%s to from revision %s to revision %s by in-place", + targetInfo.GetNamespace(), targetInfo.GetName(), + targetInfo.CurrentRevision.GetName(), + targetInfo.UpdateRevision.GetName()) + } + } else { + // if target has changes not in-place supported, recreate it + return u.GenericTargetUpdater.RecreateTarget(ctx, targetInfo) + } + return nil +} + +func (u *GenericTargetUpdater) RecreateTarget(ctx context.Context, targetInfo *targetUpdateInfo) error { + if err := u.targetControl.DeleteTarget(ctx, targetInfo.Object); err != nil { + return fmt.Errorf("fail to delete Target %s/%s when updating by recreate: %v", targetInfo.GetNamespace(), targetInfo.GetName(), err.Error()) + } + + id, err := GetInstanceID(targetInfo.Object) + if err != nil { + return fmt.Errorf("fail to get instance id for target %s/%s: %v", targetInfo.GetNamespace(), targetInfo.GetName(), err.Error()) + } + + newObj, err := NewTargetFrom(u.xsetController, u.OwnerObject, targetInfo.UpdateRevision, id) + if err != nil { + return fmt.Errorf("fail to build Target from updated revision %s: %v", targetInfo.UpdateRevision.GetName(), err.Error()) + } + if _, err := u.targetControl.CreateTarget(ctx, newObj); err != nil { + return fmt.Errorf("fail to create Target %s/%s when updating by recreate: %v", targetInfo.GetNamespace(), targetInfo.GetName(), err.Error()) + } + + u.recorder.Eventf(targetInfo.Object, + corev1.EventTypeNormal, + "UpdateTarget", + "succeed to update Target %s/%s to from revision %s to revision %s by recreate", + targetInfo.GetNamespace(), + targetInfo.GetName(), + targetInfo.CurrentRevision.GetName(), + targetInfo.UpdateRevision.GetName()) + + return nil +} + +func (u *inPlaceIfPossibleUpdater) GetTargetUpdateFinishStatus(_ context.Context, targetUpdateInfo *targetUpdateInfo) (finished bool, msg string, err error) { + if targetUpdateInfo.GetAnnotations() == nil { + return false, "no annotations for last container status", nil + } + + targetLastState := &TargetStatus{} + if lastStateJson, exist := targetUpdateInfo.GetAnnotations()[LastTargetStatusAnnotationKey]; !exist { + return false, "no target last state annotation", nil + } else if err := json.Unmarshal([]byte(lastStateJson), targetLastState); err != nil { + msg := fmt.Sprintf("malformat target last state annotation [%s]: %s", lastStateJson, err.Error()) + return false, msg, errors.New(msg) + } + + if targetLastState.ContainerStates == nil { + return true, "empty last container state recorded", nil + } + + return true, "", nil +} + +type recreateTargetUpdater struct { + GenericTargetUpdater +} + +func (u *recreateTargetUpdater) FulfillTargetUpdatedInfo(_ context.Context, _ *appsv1.ControllerRevision, _ *targetUpdateInfo) error { + return nil +} + +func (u *recreateTargetUpdater) UpgradeTarget(ctx context.Context, targetInfo *targetUpdateInfo) error { + return u.GenericTargetUpdater.RecreateTarget(ctx, targetInfo) +} + +func (u *recreateTargetUpdater) GetTargetUpdateFinishStatus(_ context.Context, targetInfo *targetUpdateInfo) (finished bool, msg string, err error) { + // Recreate policy always treat Target as update not finished + return targetInfo.IsUpdatedRevision, "", nil +} + +type replaceUpdateTargetUpdater struct { + GenericTargetUpdater +} + +func (u *replaceUpdateTargetUpdater) Setup(config *UpdateConfig, xset api.XSetObject) { + u.GenericTargetUpdater.Setup(config, xset) +} + +func (u *replaceUpdateTargetUpdater) BeginUpdateTarget(ctx context.Context, syncContext *SyncContext, targetCh chan *targetUpdateInfo) (bool, error) { + succCount, err := controllerutils.SlowStartBatch(len(targetCh), controllerutils.SlowStartInitialBatchSize, false, func(int, error) error { + targetInfo := <-targetCh + if targetInfo.ReplacePairNewTargetInfo != nil { + replacePairNewTarget := targetInfo.ReplacePairNewTargetInfo.Object + newTargetRevision, exist := replacePairNewTarget.GetLabels()[appsv1.ControllerRevisionHashLabelKey] + if exist && newTargetRevision == targetInfo.UpdateRevision.GetName() { + return nil + } + if _, exist := replacePairNewTarget.GetLabels()[TargetDeletionIndicationLabelKey]; exist { + return nil + } + + u.recorder.Eventf(targetInfo.Object, + corev1.EventTypeNormal, + "ReplaceUpdateTarget", + "label to-delete on new pair target %s/%s because it is not updated revision, current revision: %s, updated revision: %s", + replacePairNewTarget.GetNamespace(), + replacePairNewTarget.GetName(), + newTargetRevision, + syncContext.UpdatedRevision.GetName()) + patch := client.RawPatch(types.MergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:"%d"}}}`, TargetDeletionIndicationLabelKey, time.Now().UnixNano()))) + if patchErr := u.client.Patch(ctx, targetInfo.ReplacePairNewTargetInfo.Object, patch); patchErr != nil { + err := fmt.Errorf("failed to delete replace pair new target %s/%s %s", + targetInfo.ReplacePairNewTargetInfo.GetNamespace(), targetInfo.ReplacePairNewTargetInfo.GetName(), patchErr.Error()) + return err + } + } + return nil + }) + + return succCount > 0, err +} + +func (u *replaceUpdateTargetUpdater) FilterAllowOpsTargets(_ context.Context, candidates []*targetUpdateInfo, _ map[int]*appsv1alpha1.ContextDetail, _ *SyncContext, targetCh chan *targetUpdateInfo) (requeueAfter *time.Duration, err error) { + activeTargetToUpdate := filterOutPlaceHolderUpdateInfos(candidates) + for i, targetInfo := range activeTargetToUpdate { + if targetInfo.IsUpdatedRevision { + continue + } + + targetCh <- activeTargetToUpdate[i] + } + return nil, err +} + +func (u *replaceUpdateTargetUpdater) FulfillTargetUpdatedInfo(_ context.Context, _ *appsv1.ControllerRevision, _ *targetUpdateInfo) (err error) { + return +} + +func (u *replaceUpdateTargetUpdater) UpgradeTarget(ctx context.Context, targetInfo *targetUpdateInfo) error { + // add replace labels and wait to replace when syncTargets + _, replaceIndicate := targetInfo.Object.GetLabels()[TargetReplaceIndicationLabelKey] + _, replaceByUpdate := targetInfo.Object.GetLabels()[TargetReplaceByReplaceUpdateLabelKey] + if !replaceIndicate || !replaceByUpdate { + // need replace update target, label target with replace-indicate and replace-update + now := time.Now().UnixNano() + patch := client.RawPatch(types.MergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:"%v", %q: "%v"}}}`, TargetReplaceIndicationLabelKey, now, TargetReplaceByReplaceUpdateLabelKey, targetInfo.UpdateRevision.GetName()))) + if err := u.client.Patch(ctx, targetInfo.Object, patch); err != nil { + return fmt.Errorf("fail to label origin target %s/%s with replace indicate label by replaceUpdate: %s", targetInfo.GetNamespace(), targetInfo.GetName(), err.Error()) + } + u.recorder.Eventf(targetInfo.Object, + corev1.EventTypeNormal, + "UpdateTarget", + "succeed to update Target %s/%s by label to-replace", + targetInfo.GetNamespace(), + targetInfo.GetName(), + ) + } + return nil +} + +func (u *replaceUpdateTargetUpdater) GetTargetUpdateFinishStatus(_ context.Context, targetUpdateInfo *targetUpdateInfo) (finished bool, msg string, err error) { + replaceNewTargetInfo := targetUpdateInfo.ReplacePairNewTargetInfo + if replaceNewTargetInfo == nil { + return + } + + return isTargetUpdatedServiceAvailable(replaceNewTargetInfo) +} + +func (u *replaceUpdateTargetUpdater) FinishUpdateTarget(ctx context.Context, targetInfo *targetUpdateInfo) error { + ReplacePairNewTargetInfo := targetInfo.ReplacePairNewTargetInfo + if ReplacePairNewTargetInfo != nil { + if _, exist := targetInfo.GetLabels()[TargetDeletionIndicationLabelKey]; !exist { + patch := client.RawPatch(types.MergePatchType, []byte(fmt.Sprintf(`{"metadata":{"labels":{%q:"%d"}}}`, TargetDeletionIndicationLabelKey, time.Now().UnixNano()))) + if err := u.targetControl.PatchTarget(ctx, targetInfo.Object, patch); err != nil { + return fmt.Errorf("failed to delete replace pair origin target %s/%s %s", targetInfo.GetNamespace(), targetInfo.ReplacePairNewTargetInfo.GetName(), err.Error()) + } + } + } + return nil +} + +func isTargetUpdatedServiceAvailable(targetInfo *targetUpdateInfo) (finished bool, msg string, err error) { + if targetInfo.GetLabels() == nil { + return false, "no labels on target", nil + } + if targetInfo.IsInReplacing && targetInfo.ReplacePairNewTargetInfo != nil { + return false, "replace origin target", nil + } + + if _, serviceAvailable := targetInfo.GetLabels()[opslifecycle.TargetServiceAvailableLabel]; serviceAvailable { + return true, "", nil + } + + return false, "target not service available", nil +} diff --git a/xset/synccontrols/x_utils.go b/xset/synccontrols/x_utils.go new file mode 100644 index 00000000..d9327518 --- /dev/null +++ b/xset/synccontrols/x_utils.go @@ -0,0 +1,185 @@ +/* + * Copyright 2024-2025 KusionStack Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package synccontrols + +import ( + "encoding/json" + "fmt" + "strconv" + "time" + + appsv1 "k8s.io/api/apps/v1" + apimachineryvalidation "k8s.io/apimachinery/pkg/api/validation" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "kusionstack.io/kube-api/apps/v1alpha1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "kusionstack.io/kube-utils/xset/api" +) + +func GetInstanceID(target client.Object) (int, error) { + if target.GetLabels() == nil { + return -1, fmt.Errorf("no labels found for instance ID") + } + + val, exist := target.GetLabels()[TargetInstanceIDLabelKey] + if !exist { + return -1, fmt.Errorf("failed to find instance ID label %s", TargetInstanceIDLabelKey) + } + + id, err := strconv.ParseInt(val, 10, 32) + if err != nil { + // ignore invalid target instance ID + return -1, fmt.Errorf("failed to parse instance ID with value %s: %w", val, err) + } + + return int(id), nil +} + +func NewTargetFrom(setController api.XSetController, owner api.XSetObject, revision *appsv1.ControllerRevision, id int, updateFuncs ...func(client.Object) error) (client.Object, error) { + setObj := setController.EmptyXSetObject() + err := json.Unmarshal(revision.Data.Raw, setObj) + if err != nil { + return nil, err + } + + meta := setController.XSetMeta() + ownerRef := metav1.NewControllerRef(owner, meta.GroupVersionKind()) + targetObj := setController.GetXTemplate(setObj) + targetObj.SetOwnerReferences(append(targetObj.GetOwnerReferences(), *ownerRef)) + targetObj.SetNamespace(owner.GetNamespace()) + targetObj.SetGenerateName(GetTargetsPrefix(owner.GetName())) + + labels := targetObj.GetLabels() + labels[TargetInstanceIDLabelKey] = fmt.Sprintf("%d", id) + labels[appsv1.ControllerRevisionHashLabelKey] = revision.GetName() + controlByKusionStack(targetObj) + + for _, fn := range updateFuncs { + if err := fn(targetObj); err != nil { + return targetObj, err + } + } + + return targetObj, nil +} + +func RealValue(val *int32) int32 { + if val == nil { + return 0 + } + + return *val +} + +const ConditionUpdatePeriodBackOff = 30 * time.Second + +func AddOrUpdateCondition(status *api.XSetStatus, conditionType api.XSetConditionType, err error, reason, message string) { + condStatus := metav1.ConditionTrue + if err != nil { + condStatus = metav1.ConditionFalse + } + + existCond := GetCondition(status, string(conditionType)) + if existCond != nil && existCond.Reason == reason && existCond.Status == condStatus { + now := metav1.Now() + if now.Sub(existCond.LastTransitionTime.Time) < ConditionUpdatePeriodBackOff { + return + } + } + + cond := NewCondition(string(conditionType), condStatus, reason, message) + SetCondition(status, cond) +} + +func NewCondition(condType string, status metav1.ConditionStatus, reason, msg string) *metav1.Condition { + return &metav1.Condition{ + Type: condType, + Status: status, + LastTransitionTime: metav1.Now(), + Reason: reason, + Message: msg, + } +} + +// SetCondition adds/replaces the given condition in the replicaset status. If the condition that we +// are about to add already exists and has the same status and reason then we are not going to update. +func SetCondition(status *api.XSetStatus, condition *metav1.Condition) { + currentCond := GetCondition(status, condition.Type) + if currentCond != nil && currentCond.Status == condition.Status && currentCond.Reason == condition.Reason && currentCond.LastTransitionTime == condition.LastTransitionTime { + return + } + newConditions := filterOutCondition(status.Conditions, condition.Type) + newConditions = append(newConditions, *condition) + status.Conditions = newConditions +} + +// GetCondition returns a inplace set condition with the provided type if it exists. +func GetCondition(status *api.XSetStatus, condType string) *metav1.Condition { + for _, c := range status.Conditions { + if c.Type == condType { + return &c + } + } + return nil +} + +func GetTargetsPrefix(controllerName string) string { + // use the dash (if the name isn't too long) to make the target name a bit prettier + prefix := fmt.Sprintf("%s-", controllerName) + if len(apimachineryvalidation.NameIsDNSSubdomain(prefix, true)) != 0 { + prefix = controllerName + } + return prefix +} + +func IsTargetUpdatedRevision(target client.Object, revision string) bool { + if target.GetLabels() == nil { + return false + } + + return target.GetLabels()[appsv1.ControllerRevisionHashLabelKey] == revision +} + +func ObjectKeyString(obj client.Object) string { + if obj.GetNamespace() == "" { + return obj.GetName() + } + return obj.GetNamespace() + "/" + obj.GetName() +} + +// filterOutCondition returns a new slice of replicaset conditions without conditions with the provided type. +func filterOutCondition(conditions []metav1.Condition, condType string) []metav1.Condition { + var newConditions []metav1.Condition + for _, c := range conditions { + if c.Type == condType { + continue + } + newConditions = append(newConditions, c) + } + return newConditions +} + +func controlByKusionStack(obj client.Object) { + if obj.GetLabels() == nil { + obj.SetLabels(map[string]string{}) + } + + if v, ok := obj.GetLabels()[v1alpha1.ControlledByKusionStackLabelKey]; !ok || v != "true" { + obj.GetLabels()[v1alpha1.ControlledByKusionStackLabelKey] = "true" + } +} diff --git a/xset/xcontrol/target_control.go b/xset/xcontrol/target_control.go new file mode 100644 index 00000000..f9ba197c --- /dev/null +++ b/xset/xcontrol/target_control.go @@ -0,0 +1,217 @@ +/* + * Copyright 2024-2025 KusionStack Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package xcontrol + +import ( + "context" + "fmt" + "reflect" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + + "kusionstack.io/kube-utils/controller/expectations" + "kusionstack.io/kube-utils/controller/mixin" + refmanagerutil "kusionstack.io/kube-utils/controller/refmanager" + "kusionstack.io/kube-utils/controller/utils" + "kusionstack.io/kube-utils/xset/api" +) + +const ( + FieldIndexOwnerRefUID = "ownerRefUID" +) + +type TargetControl interface { + GetFilteredTargets(ctx context.Context, selector *metav1.LabelSelector, owner api.XSetObject) ([]client.Object, error) + CreateTarget(ctx context.Context, target client.Object) (client.Object, error) + DeleteTarget(ctx context.Context, target client.Object) error + UpdateTarget(ctx context.Context, target client.Object) error + PatchTarget(ctx context.Context, target client.Object, patch client.Patch) error + OrphanTarget(xset api.XSetObject, target client.Object) error + AdoptTarget(xset api.XSetObject, target client.Object) error +} + +type targetControl struct { + client client.Client + schema *runtime.Scheme + + xsetController api.XSetController + xGVK schema.GroupVersionKind + cacheExpectation *expectations.CacheExpectation +} + +func NewTargetControl(mixin *mixin.ReconcilerMixin, xsetController api.XSetController, cacheExpectation *expectations.CacheExpectation) (TargetControl, error) { + if err := setUpCache(mixin.Cache, xsetController); err != nil { + return nil, err + } + + xMeta := xsetController.XMeta() + gvk := xMeta.GroupVersionKind() + return &targetControl{ + client: mixin.Client, + schema: mixin.Scheme, + xsetController: xsetController, + xGVK: gvk, + cacheExpectation: cacheExpectation, + }, nil +} + +func (r *targetControl) GetFilteredTargets(ctx context.Context, selector *metav1.LabelSelector, owner api.XSetObject) ([]client.Object, error) { + targetList := r.xsetController.EmptyXObjectList() + if err := r.client.List(ctx, targetList, &client.ListOptions{ + Namespace: owner.GetNamespace(), + FieldSelector: fields.OneTermEqualSelector(FieldIndexOwnerRefUID, string(owner.GetUID())), + }); err != nil { + return nil, err + } + + targetListVal := reflect.Indirect(reflect.ValueOf(targetList)) + itemsVal := targetListVal.FieldByName("Items") + if !itemsVal.IsValid() { + return nil, fmt.Errorf("target list items is invalid") + } + + var items []client.Object + if itemsVal.Kind() == reflect.Slice { + items = make([]client.Object, itemsVal.Len()) + for i := 0; i < itemsVal.Len(); i++ { + itemVal := itemsVal.Index(i).Addr().Interface() + items[i] = itemVal.(client.Object) + } + } else { + return nil, fmt.Errorf("target list items is invalid") + } + + // todo filterOutInactiveTargets + targets, err := r.getTargets(items, selector, owner) + return targets, err +} + +func (r *targetControl) CreateTarget(ctx context.Context, target client.Object) (client.Object, error) { + if err := r.client.Create(ctx, target); err != nil { + return nil, fmt.Errorf("failed to create target: %s", err.Error()) + } else { + return target, r.cacheExpectation.ExpectCreation(r.xGVK, target.GetNamespace(), target.GetName()) + } +} + +func (r *targetControl) DeleteTarget(ctx context.Context, target client.Object) error { + if err := r.client.Delete(ctx, target); err != nil { + return fmt.Errorf("failed to delete target: %s", err.Error()) + } else { + return r.cacheExpectation.ExpectDeletion(r.xGVK, target.GetNamespace(), target.GetName()) + } +} + +func (r *targetControl) UpdateTarget(ctx context.Context, target client.Object) error { + if err := r.client.Update(ctx, target); err != nil { + return fmt.Errorf("failed to update target: %s", err.Error()) + } else { + return r.cacheExpectation.ExpectUpdation(r.xGVK, target.GetNamespace(), target.GetName(), target.GetResourceVersion()) + } +} + +func (r *targetControl) PatchTarget(ctx context.Context, target client.Object, patch client.Patch) error { + if err := r.client.Patch(ctx, target, patch); err != nil { + return fmt.Errorf("failed to patch target: %s", err.Error()) + } else { + return r.cacheExpectation.ExpectUpdation(r.xGVK, target.GetNamespace(), target.GetName(), target.GetResourceVersion()) + } +} + +func (r *targetControl) OrphanTarget(xset api.XSetObject, target client.Object) error { + spec := r.xsetController.GetXSetSpec(xset) + if spec.Selector.MatchLabels == nil { + return nil + } + + if target.GetLabels() == nil { + target.SetLabels(make(map[string]string)) + } + if target.GetAnnotations() == nil { + target.SetAnnotations(make(map[string]string)) + } + + refWriter := refmanagerutil.NewOwnerRefWriter(r.client) + if err := refWriter.Release(context.TODO(), xset, target); err != nil { + return fmt.Errorf("failed to orphan target: %s", err.Error()) + } + + return r.cacheExpectation.ExpectUpdation(r.xGVK, target.GetNamespace(), target.GetName(), target.GetResourceVersion()) +} + +func (r *targetControl) AdoptTarget(xset api.XSetObject, target client.Object) error { + spec := r.xsetController.GetXSetSpec(xset) + if spec.Selector.MatchLabels == nil { + return nil + } + + refWriter := refmanagerutil.NewOwnerRefWriter(r.client) + matcher, err := refmanagerutil.LabelSelectorAsMatch(spec.Selector) + if err != nil { + return fmt.Errorf("fail to create labelSelector matcher: %s", err.Error()) + } + refManager := refmanagerutil.NewObjectControllerRefManager(refWriter, xset, xset.GetObjectKind().GroupVersionKind(), matcher) + + if _, err = refManager.Claim(context.TODO(), target); err != nil { + return fmt.Errorf("failed to adopt target: %s", err.Error()) + } + + return r.cacheExpectation.ExpectUpdation(r.xGVK, target.GetNamespace(), target.GetName(), target.GetResourceVersion()) +} + +func (r *targetControl) getTargets(candidates []client.Object, selector *metav1.LabelSelector, xset api.XSetObject) ([]client.Object, error) { + // Use RefManager to adopt/orphan as needed. + writer := refmanagerutil.NewOwnerRefWriter(r.client) + matcher, err := refmanagerutil.LabelSelectorAsMatch(selector) + if err != nil { + return nil, fmt.Errorf("fail to create labelSelector matcher: %s", err.Error()) + } + refManager := refmanagerutil.NewObjectControllerRefManager(writer, xset, xset.GetObjectKind().GroupVersionKind(), matcher) + + var claimObjs []client.Object + var errList []error + for _, obj := range candidates { + ok, err := refManager.Claim(context.TODO(), obj) + if err != nil { + errList = append(errList, err) + continue + } + if ok { + claimObjs = append(claimObjs, obj) + } + } + + return claimObjs, utils.AggregateErrors(errList) +} + +func setUpCache(cache cache.Cache, controller api.XSetController) error { + if err := cache.IndexField(context.TODO(), controller.EmptyXObject(), FieldIndexOwnerRefUID, func(object client.Object) []string { + ownerRef := metav1.GetControllerOf(object) + if ownerRef == nil || ownerRef.Kind != controller.XSetMeta().Kind { + return nil + } + return []string{string(ownerRef.UID)} + }); err != nil { + return fmt.Errorf("failed to index by field %s: %s", FieldIndexOwnerRefUID, err.Error()) + } + return nil +} diff --git a/xset/xset_controller.go b/xset/xset_controller.go new file mode 100644 index 00000000..7de410b6 --- /dev/null +++ b/xset/xset_controller.go @@ -0,0 +1,244 @@ +package xset + +import ( + "context" + "fmt" + "time" + + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/clock" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + "kusionstack.io/kube-utils/controller/expectations" + "kusionstack.io/kube-utils/controller/mixin" + "kusionstack.io/kube-utils/controller/revision" + controllerutils "kusionstack.io/kube-utils/controller/utils" + "kusionstack.io/kube-utils/xset/api" + "kusionstack.io/kube-utils/xset/resourcecontexts" + "kusionstack.io/kube-utils/xset/revisionadapter" + "kusionstack.io/kube-utils/xset/synccontrols" + "kusionstack.io/kube-utils/xset/xcontrol" +) + +type xSetCommonReconciler struct { + mixin.ReconcilerMixin + + XSetController api.XSetController + meta metav1.TypeMeta + finalizerName string + xsetGVK schema.GroupVersionKind + + // reconcile logic helpers + cacheExpectation *expectations.CacheExpectation + targetControl xcontrol.TargetControl + syncControl synccontrols.SyncControl + revisionManager *revision.RevisionManager +} + +func SetUpWithManager(mgr ctrl.Manager, xsetController api.XSetController) error { + reconcilerMixin := mixin.NewReconcilerMixin(xsetController.ControllerName(), mgr) + xsetMeta := xsetController.XSetMeta() + xsetGVK := xsetMeta.GroupVersionKind() + targetMeta := xsetController.XMeta() + + cacheExpectations := expectations.NewxCacheExpectations(reconcilerMixin.Client, reconcilerMixin.Scheme, clock.RealClock{}) + cacheExpectation, err := cacheExpectations.CreateExpectations(xsetController.ControllerName()) + if err != nil { + return fmt.Errorf("failed to get expectations: %s", err.Error()) + } + + targetControl, err := xcontrol.NewTargetControl(reconcilerMixin, xsetController, cacheExpectation) + syncControl := synccontrols.NewRealSyncControl(reconcilerMixin, xsetController, targetControl, cacheExpectation) + revisionManager := revision.NewRevisionManager(reconcilerMixin.Client, reconcilerMixin.Scheme, revisionadapter.NewRevisionOwnerAdapter(xsetController, targetControl)) + reconciler := &xSetCommonReconciler{ + targetControl: targetControl, + ReconcilerMixin: *reconcilerMixin, + XSetController: xsetController, + meta: xsetController.XSetMeta(), + finalizerName: xsetController.FinalizerName(), + syncControl: syncControl, + revisionManager: revisionManager, + cacheExpectation: cacheExpectation, + xsetGVK: xsetGVK, + } + + c, err := controller.New(xsetController.ControllerName(), mgr, controller.Options{ + MaxConcurrentReconciles: 5, + Reconciler: reconciler, + }) + if err != nil { + return fmt.Errorf("failed to create controller: %s", err.Error()) + } + + if err := c.Watch(&source.Kind{Type: xsetController.EmptyXSetObject()}, &handler.EnqueueRequestForObject{}); err != nil { + return fmt.Errorf("failed to watch %s: %s", xsetController.XSetMeta().Kind, err.Error()) + } + + if err := c.Watch(&source.Kind{Type: xsetController.EmptyXObject()}, &handler.EnqueueRequestForOwner{ + IsController: true, + OwnerType: xsetController.EmptyXSetObject(), + }); err != nil { + return fmt.Errorf("failed to watch %s: %s", targetMeta.Kind, err.Error()) + } + + return nil +} + +func (r *xSetCommonReconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { + kind := r.meta.Kind + key := req.String() + logger := r.Logger.WithValues(kind, key) + instance := r.XSetController.EmptyXSetObject() + if err := r.Client.Get(ctx, req.NamespacedName, instance); err != nil { + if !errors.IsNotFound(err) { + logger.Error(err, "failed to find object") + return reconcile.Result{}, err + } + + logger.Info("object deleted") + return ctrl.Result{}, r.cacheExpectation.ExpectDeletion(r.xsetGVK, req.Namespace, req.Name) + } + + // if cacheExpectation not fulfilled, shortcut this reconciling till informer cache is updated. + if satisfied := r.cacheExpectation.FulFilledFor(r.xsetGVK, req.Namespace, req.Name); !satisfied { + logger.Info("not satisfied to reconcile") + return ctrl.Result{RequeueAfter: 30 * time.Second}, nil + } + + if instance.GetDeletionTimestamp() != nil { + // todo, why reclaim ownerReferences before target deletion? + if err := r.ensureReclaimTargetOwnerReferences(ctx, instance); err != nil { + // reclaim targets ownerReferences before remove finalizers + return ctrl.Result{}, err + } + if err := r.ensureReclaimTargetsDeletion(ctx, instance); err != nil { + // reclaim targets deletion before remove finalizers + return ctrl.Result{}, err + } + if controllerutil.ContainsFinalizer(instance, r.finalizerName) { + // reclaim owner IDs in ResourceContext + if err := resourcecontexts.UpdateToTargetContext(r.XSetController, r.Client, r.cacheExpectation, instance, nil); err != nil { + return ctrl.Result{}, err + } + if err := controllerutils.RemoveFinalizer(ctx, r.Client, instance, r.finalizerName); err != nil { + return ctrl.Result{}, err + } + } + + return ctrl.Result{}, nil + } + + if !controllerutil.ContainsFinalizer(instance, r.finalizerName) { + return ctrl.Result{}, controllerutils.AddFinalizer(context.TODO(), r.Client, instance, r.finalizerName) + } + + currentRevision, updatedRevision, revisions, collisionCount, _, err := r.revisionManager.ConstructRevisions(instance, false) + if err != nil { + return ctrl.Result{}, fmt.Errorf("fail to construct revision for %s %s: %s", kind, key, err.Error()) + } + + xsetStatus := r.XSetController.GetXSetStatus(instance) + newStatus := xsetStatus.DeepCopy() + newStatus.UpdatedRevision = updatedRevision.Name + newStatus.CurrentRevision = currentRevision.Name + newStatus.CollisionCount = collisionCount + syncContext := &synccontrols.SyncContext{ + Revisions: revisions, + CurrentRevision: currentRevision, + UpdatedRevision: updatedRevision, + NewStatus: newStatus, + } + + requeueAfter, syncErr := r.doSync(ctx, instance, syncContext) + if syncErr != nil { + logger.Error(syncErr, "failed to sync") + } + + newStatus = r.syncControl.CalculateStatus(ctx, instance, syncContext) + // update status anyway + if err := r.updateStatus(ctx, instance, newStatus); err != nil { + return requeueResult(requeueAfter), fmt.Errorf("fail to update status of %s %s: %s", kind, req, err.Error()) + } + return requeueResult(requeueAfter), syncErr +} + +func (r *xSetCommonReconciler) doSync(ctx context.Context, instance api.XSetObject, syncContext *synccontrols.SyncContext) (*time.Duration, error) { + synced, err := r.syncControl.SyncTargets(ctx, instance, syncContext) + if err != nil || synced { + return nil, err + } + + err = r.syncControl.Replace(ctx, instance, syncContext) + if err != nil { + return nil, err + } + + _, scaleRequeueAfter, scaleErr := r.syncControl.Scale(ctx, instance, syncContext) + _, updateRequeueAfter, updateErr := r.syncControl.Update(ctx, instance, syncContext) + + err = controllerutils.AggregateErrors([]error{scaleErr, updateErr}) + if updateRequeueAfter != nil && (scaleRequeueAfter == nil || *updateRequeueAfter < *scaleRequeueAfter) { + return updateRequeueAfter, err + } + return scaleRequeueAfter, err +} + +func (r *xSetCommonReconciler) ensureReclaimTargetOwnerReferences(ctx context.Context, instance api.XSetObject) error { + xspec := r.XSetController.GetXSetSpec(instance) + targets, err := r.targetControl.GetFilteredTargets(ctx, xspec.Selector, instance) + if err != nil { + return fmt.Errorf("fail to get filtered Targets: %s", err.Error()) + } + for i := range targets { + target := targets[i] + targetOwner := target.GetOwnerReferences() + if len(targetOwner) == 0 { + continue + } + var newOwnerRefs []metav1.OwnerReference + for j := range targetOwner { + newOwnerRefs = append(newOwnerRefs, targetOwner[j]) + } + if len(newOwnerRefs) != len(targetOwner) { + target.SetOwnerReferences(newOwnerRefs) + if err := r.targetControl.UpdateTarget(ctx, target); err != nil { + return err + } + } + } + return nil +} + +func (r *xSetCommonReconciler) ensureReclaimTargetsDeletion(ctx context.Context, instance api.XSetObject) error { + xspec := r.XSetController.GetXSetSpec(instance) + targets, err := r.targetControl.GetFilteredTargets(ctx, xspec.Selector, instance) + if err != nil { + return fmt.Errorf("fail to get filtered Targets: %s", err.Error()) + } + return synccontrols.BatchDelete(ctx, r.targetControl, targets) +} + +func (r *xSetCommonReconciler) updateStatus(ctx context.Context, instance api.XSetObject, status *api.XSetStatus) error { + r.XSetController.SetXSetStatus(instance, status) + if err := r.Client.Status().Update(ctx, instance); err != nil { + return fmt.Errorf("fail to update status of %s: %s", instance.GetName(), err.Error()) + } + return r.cacheExpectation.ExpectUpdation(r.xsetGVK, instance.GetNamespace(), instance.GetName(), instance.GetResourceVersion()) +} + +func requeueResult(requeueTime *time.Duration) reconcile.Result { + if requeueTime != nil { + if *requeueTime == 0 { + return reconcile.Result{Requeue: true} + } + return reconcile.Result{RequeueAfter: *requeueTime} + } + return reconcile.Result{} +} From 2808e51e0e2ef43b404debffe7d816a75023d334 Mon Sep 17 00:00:00 2001 From: "yuyinglu.yyl" Date: Mon, 16 Jun 2025 17:07:41 +0800 Subject: [PATCH 2/5] some adjustment for pr comments --- controller/revision/revision_manager.go | 419 ------------------ controller/revision/revision_manager_test.go | 333 -------------- controller/utils/error.go | 40 -- controller/utils/error_test.go | 54 --- controller/utils/finalizer.go | 64 --- .../revision_owner.go} | 90 ++-- xset/synccontrols/sync_control.go | 9 +- xset/synccontrols/x_scale.go | 3 +- xset/xcontrol/target_control.go | 4 +- xset/xset_controller.go | 75 ++-- 10 files changed, 97 insertions(+), 994 deletions(-) delete mode 100644 controller/revision/revision_manager.go delete mode 100644 controller/revision/revision_manager_test.go delete mode 100644 controller/utils/error.go delete mode 100644 controller/utils/error_test.go delete mode 100644 controller/utils/finalizer.go rename xset/{revisionadapter/revision_adapter.go => revisionowner/revision_owner.go} (53%) diff --git a/controller/revision/revision_manager.go b/controller/revision/revision_manager.go deleted file mode 100644 index 93eb4c62..00000000 --- a/controller/revision/revision_manager.go +++ /dev/null @@ -1,419 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. -Copyright 2023 The KusionStack Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package revision - -import ( - "bytes" - "context" - "fmt" - "hash/fnv" - "sort" - "strconv" - - apps "k8s.io/api/apps/v1" - apiequality "k8s.io/apimachinery/pkg/api/equality" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/rand" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/apiutil" - - "kusionstack.io/kube-utils/controller/refmanager" -) - -const ControllerRevisionHashLabel = "controller.kubernetes.io/hash" - -type OwnerAdapter interface { - GetSelector(obj metav1.Object) *metav1.LabelSelector - GetCollisionCount(obj metav1.Object) *int32 - GetHistoryLimit(obj metav1.Object) int32 - GetPatch(obj metav1.Object) ([]byte, error) - GetCurrentRevision(obj metav1.Object) string - IsInUsed(obj metav1.Object, controllerRevision string) bool -} - -func NewRevisionManager(client client.Client, scheme *runtime.Scheme, ownerGetter OwnerAdapter) *RevisionManager { - return &RevisionManager{ - Client: client, - scheme: scheme, - ownerGetter: ownerGetter, - } -} - -type RevisionManager struct { - client.Client - - scheme *runtime.Scheme - ownerGetter OwnerAdapter -} - -// controlledHistories returns all ControllerRevisions controlled by the given DaemonSet. -// This also reconciles ControllerRef by adopting/orphaning. -// Note that returned histories are pointers to objects in the cache. -// If you want to modify one, you need to deep-copy it first. -func controlledHistories(c client.Client, owner client.Object, labelSelector *metav1.LabelSelector) ([]*apps.ControllerRevision, error) { - // List all histories to include those that don't match the selector anymore - // but have a ControllerRef pointing to the controller. - selector, err := metav1.LabelSelectorAsSelector(labelSelector) - if err != nil { - return nil, err - } - histories := &apps.ControllerRevisionList{} - if err := c.List(context.TODO(), histories, &client.ListOptions{Namespace: owner.GetNamespace(), LabelSelector: selector}); err != nil { - return nil, err - } - - // Use ControllerRefManager to adopt/orphan as needed. - writer := refmanager.NewOwnerRefWriter(c) - matcher, err := refmanager.LabelSelectorAsMatch(labelSelector) - if err != nil { - return nil, fmt.Errorf("fail to create labelSelector matcher: %s", err.Error()) - } - - refManager := refmanager.NewObjectControllerRefManager(writer, owner, owner.GetObjectKind().GroupVersionKind(), matcher) - - mts := make([]metav1.Object, len(histories.Items)) - for i := range histories.Items { - mts[i] = &histories.Items[i] - } - claims, err := refManager.ClaimAllOf(context.TODO(), mts) - if err != nil { - return nil, err - } - - claimHistories := make([]*apps.ControllerRevision, len(claims)) - for i, mt := range claims { - claimHistories[i] = mt.(*apps.ControllerRevision) - } - - return claimHistories, nil -} - -// ConstructRevisions returns the current and update ControllerRevisions for set. It also -// returns a collision count that records the number of name collisions set saw when creating -// new ControllerRevisions. This count is incremented on every name collision and is used in -// building the ControllerRevision names for name collision avoidance. This method may create -// a new revision, or modify the Revision of an existing revision if an update to set is detected. -// This method expects that revisions is sorted when supplied. -func (rm *RevisionManager) ConstructRevisions(set client.Object, dryRun bool) (*apps.ControllerRevision, *apps.ControllerRevision, []*apps.ControllerRevision, *int32, bool, error) { - var currentRevision, updateRevision *apps.ControllerRevision - revisions, err := controlledHistories(rm.Client, set, rm.ownerGetter.GetSelector(set)) - if err != nil { - return currentRevision, updateRevision, nil, rm.ownerGetter.GetCollisionCount(set), false, err - } - - SortControllerRevisions(revisions) - if cleanedRevision, err := rm.cleanExpiredRevision(set, &revisions); err != nil { - return currentRevision, updateRevision, nil, rm.ownerGetter.GetCollisionCount(set), false, err - } else { - revisions = *cleanedRevision - } - - collisionCount := new(int32) - if rm.ownerGetter.GetCollisionCount(set) != nil { - collisionCount = rm.ownerGetter.GetCollisionCount(set) - } - // create a new revision from the current set - updateRevision, err = rm.newRevision(set, nextRevision(revisions), collisionCount) - if err != nil { - return nil, nil, nil, collisionCount, false, err - } - - // find any equivalent revisions - equalRevisions := FindEqualRevisions(revisions, updateRevision) - equalCount := len(equalRevisions) - revisionCount := len(revisions) - - createNewRevision := false - if equalCount > 0 && EqualRevision(revisions[revisionCount-1], equalRevisions[equalCount-1]) { - // if the equivalent revision is immediately prior the update revision has not changed - updateRevision = revisions[revisionCount-1] - } else if equalCount > 0 { - // if the equivalent revision is not immediately prior we will roll back by incrementing the - // Revision of the equivalent revision - equalRevisions[equalCount-1].Revision = updateRevision.Revision - rvc := equalRevisions[equalCount-1] - err = rm.Update(context.TODO(), rvc) - if err != nil { - return nil, nil, nil, collisionCount, false, err - } - equalRevisions[equalCount-1] = rvc - updateRevision = equalRevisions[equalCount-1] - } else { - if !dryRun { - // if there is no equivalent revision we create a new one - updateRevision, err = rm.createControllerRevision(context.TODO(), set, updateRevision, collisionCount) - if err != nil { - return nil, nil, nil, collisionCount, false, err - } - } - - revisions = append(revisions, updateRevision) - createNewRevision = true - } - - // attempt to find the revision that corresponds to the current revision - for i := range revisions { - if revisions[i].Name == rm.ownerGetter.GetCurrentRevision(set) { - currentRevision = revisions[i] - } - } - - // if the current revision is nil we initialize the history by setting it to the update revision - if currentRevision == nil { - currentRevision = updateRevision - } - - return currentRevision, updateRevision, revisions, collisionCount, createNewRevision, nil -} - -func (rm *RevisionManager) cleanExpiredRevision(set metav1.Object, sortedRevisions *[]*apps.ControllerRevision) (*[]*apps.ControllerRevision, error) { - limit := int(rm.ownerGetter.GetHistoryLimit(set)) - - // reserve 2 extra unused revisions for diagnose - exceedNum := len(*sortedRevisions) - limit - 2 - if exceedNum <= 0 { - return sortedRevisions, nil - } - - var cleanedRevisions []*apps.ControllerRevision - for _, revision := range *sortedRevisions { - if exceedNum == 0 || rm.ownerGetter.IsInUsed(set, revision.Name) { - cleanedRevisions = append(cleanedRevisions, revision) - continue - } - - if err := rm.Delete(context.TODO(), revision); err != nil { - return sortedRevisions, err - } - - exceedNum-- - } - - return &cleanedRevisions, nil -} - -func (rm *RevisionManager) createControllerRevision(ctx context.Context, parent metav1.Object, revision *apps.ControllerRevision, collisionCount *int32) (*apps.ControllerRevision, error) { - if collisionCount == nil { - return nil, fmt.Errorf("collisionCount should not be nil") - } - - // Clone the input - clone := revision.DeepCopy() - - var err error - // Continue to attempt to create the revision updating the name with a new hash on each iteration - for { - hash := hashControllerRevision(revision, collisionCount) - // Update the revisions name - clone.Name = controllerRevisionName(parent.GetName(), hash) - err = rm.Create(ctx, clone) - if errors.IsAlreadyExists(err) { - exists := &apps.ControllerRevision{} - err := rm.Get(ctx, types.NamespacedName{Namespace: clone.Namespace, Name: clone.Name}, exists) - if err != nil { - return nil, err - } - if bytes.Equal(exists.Data.Raw, clone.Data.Raw) { - return exists, nil - } - *collisionCount++ - continue - } - return clone, err - } -} - -// controllerRevisionName returns the Name for a ControllerRevision in the form prefix-hash. If the length -// of prefix is greater than 223 bytes, it is truncated to allow for a name that is no larger than 253 bytes. -func controllerRevisionName(prefix, hash string) string { - if len(prefix) > 223 { - prefix = prefix[:223] - } - - return fmt.Sprintf("%s-%s", prefix, hash) -} - -// hashControllerRevision hashes the contents of revision's Data using FNV hashing. If probe is not nil, the byte value -// of probe is added written to the hash as well. The returned hash will be a safe encoded string to avoid bad words. -func hashControllerRevision(revision *apps.ControllerRevision, probe *int32) string { - hf := fnv.New32() - if len(revision.Data.Raw) > 0 { - hf.Write(revision.Data.Raw) - } - if probe != nil { - hf.Write([]byte(strconv.FormatInt(int64(*probe), 10))) - } - return rand.SafeEncodeString(fmt.Sprint(hf.Sum32())) -} - -// newRevision creates a new ControllerRevision containing a patch that reapplies the target state of set. -// The Revision of the returned ControllerRevision is set to revision. If the returned error is nil, the returned -// ControllerRevision is valid. StatefulSet revisions are stored as patches that re-apply the current state of set -// to a new StatefulSet using a strategic merge patch to replace the saved state of the new StatefulSet. -func (rm *RevisionManager) newRevision(set metav1.Object, revision int64, collisionCount *int32) (*apps.ControllerRevision, error) { - patch, err := rm.ownerGetter.GetPatch(set) - if err != nil { - return nil, err - } - - runtimeObj, ok := set.(runtime.Object) - if !ok { - return nil, fmt.Errorf("revision owner %s/%s does not implement runtime Object interface", set.GetNamespace(), set.GetName()) - } - gvk, err := apiutil.GVKForObject(runtimeObj, rm.scheme) - if err != nil { - return nil, err - } - - revisionLabels := map[string]string{} - if selector := rm.ownerGetter.GetSelector(set); selector != nil { - for k, v := range selector.MatchLabels { - revisionLabels[k] = v - } - } - - cr := newControllerRevision(set, - gvk, - revisionLabels, - runtime.RawExtension{Raw: patch}, - revision, - collisionCount) - if err != nil { - return nil, err - } - - cr.Namespace = set.GetNamespace() - - return cr, nil -} - -// nextRevision finds the next valid revision number based on revisions. If the length of revisions -// is 0 this is 1. Otherwise, it is 1 greater than the largest revision's Revision. This method -// assumes that revisions has been sorted by Revision. -func nextRevision(revisions []*apps.ControllerRevision) int64 { - count := len(revisions) - if count <= 0 { - return 1 - } - return revisions[count-1].Revision + 1 -} - -// SortControllerRevisions sorts revisions by their Revision. -func SortControllerRevisions(revisions []*apps.ControllerRevision) { - sort.Sort(byRevision(revisions)) -} - -// byRevision implements sort.Interface to allow ControllerRevisions to be sorted by Revision. -type byRevision []*apps.ControllerRevision - -func (br byRevision) Len() int { - return len(br) -} - -func (br byRevision) Less(i, j int) bool { - return br[i].Revision < br[j].Revision -} - -func (br byRevision) Swap(i, j int) { - br[i], br[j] = br[j], br[i] -} - -// EqualRevision returns true if lhs and rhs are either both nil, or both have same labels and annotations, or bath point -// to non-nil ControllerRevisions that contain semantically equivalent data. Otherwise this method returns false. -func EqualRevision(lhs, rhs *apps.ControllerRevision) bool { - var lhsHash, rhsHash *uint32 - if lhs == nil || rhs == nil { - return lhs == rhs - } - - if hs, found := lhs.Labels[ControllerRevisionHashLabel]; found { - hash, err := strconv.ParseInt(hs, 10, 32) - if err == nil { - lhsHash = new(uint32) - *lhsHash = uint32(hash) - } - } - if hs, found := rhs.Labels[ControllerRevisionHashLabel]; found { - hash, err := strconv.ParseInt(hs, 10, 32) - if err == nil { - rhsHash = new(uint32) - *rhsHash = uint32(hash) - } - } - if lhsHash != nil && rhsHash != nil && *lhsHash != *rhsHash { - return false - } - return bytes.Equal(lhs.Data.Raw, rhs.Data.Raw) && apiequality.Semantic.DeepEqual(lhs.Data.Object, rhs.Data.Object) -} - -// FindEqualRevisions returns all ControllerRevisions in revisions that are equal to needle using EqualRevision as the -// equality test. The returned slice preserves the order of revisions. -func FindEqualRevisions(revisions []*apps.ControllerRevision, needle *apps.ControllerRevision) []*apps.ControllerRevision { - var eq []*apps.ControllerRevision - for i := range revisions { - if EqualRevision(revisions[i], needle) { - eq = append(eq, revisions[i]) - } - } - return eq -} - -// newControllerRevision returns a ControllerRevision with a ControllerRef pointing to parent and indicating that -// parent is of parentKind. The ControllerRevision has labels matching template labels, contains Data equal to data, and -// has a Revision equal to revision. The collisionCount is used when creating the name of the ControllerRevision -// so the name is likely unique. If the returned error is nil, the returned ControllerRevision is valid. If the -// returned error is not nil, the returned ControllerRevision is invalid for use. -func newControllerRevision(parent metav1.Object, - parentKind schema.GroupVersionKind, - templateLabels map[string]string, - data runtime.RawExtension, - revision int64, - collisionCount *int32, -) *apps.ControllerRevision { - labelMap := make(map[string]string) - for k, v := range templateLabels { - labelMap[k] = v - } - blockOwnerDeletion := true - isController := true - cr := &apps.ControllerRevision{ - ObjectMeta: metav1.ObjectMeta{ - Labels: labelMap, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: parentKind.GroupVersion().String(), - Kind: parentKind.Kind, - Name: parent.GetName(), - UID: parent.GetUID(), - BlockOwnerDeletion: &blockOwnerDeletion, - Controller: &isController, - }, - }, - }, - Data: data, - Revision: revision, - } - hash := hashControllerRevision(cr, collisionCount) - cr.Name = controllerRevisionName(parent.GetName(), hash) - cr.Labels[ControllerRevisionHashLabel] = hash - return cr -} diff --git a/controller/revision/revision_manager_test.go b/controller/revision/revision_manager_test.go deleted file mode 100644 index 1617a8fe..00000000 --- a/controller/revision/revision_manager_test.go +++ /dev/null @@ -1,333 +0,0 @@ -/* -Copyright 2023 The KusionStack Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package revision - -import ( - "context" - "encoding/json" - "fmt" - "testing" - "time" - - . "github.com/onsi/gomega" - - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/client/fake" -) - -var ( - c client.Client - g *WithT - - selectedLabels = map[string]string{"test": "foo"} - selector, _ = metav1.ParseToLabelSelector("test=foo") -) - -func TestRevisionConstruction(t *testing.T) { - g = NewGomegaWithT(t) - testcase := "test-revision-construction" - schema := runtime.NewScheme() - corev1.AddToScheme(schema) - appsv1.AddToScheme(schema) - c = fake.NewClientBuilder().WithScheme(schema).Build() - g.Expect(createNamespace(c, testcase)).Should(Succeed()) - - deploy := &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: testcase, - Name: testcase, - }, - Spec: appsv1.DeploymentSpec{ - Selector: selector, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: selectedLabels, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "test", - Image: "nginx:v1", - }, - }, - }, - }, - }, - } - g.Expect(c.Create(context.TODO(), deploy)).Should(Succeed()) - - adapter := &OwnerAdapterImpl{ - name: testcase, - selector: selector, - selectedLabels: selectedLabels, - collisionCount: 0, - historyLimit: 2, - currentRevision: "test", - inUsed: true, - } - - revisionManager := NewRevisionManager(c, schema, adapter) - currentRevision, updatedRevision, revisionList, collisionCount, createNewRevision, err := revisionManager.ConstructRevisions(deploy, false) - g.Expect(err).Should(Succeed()) - g.Expect(createNewRevision).Should(BeTrue()) - g.Expect(collisionCount).ShouldNot(BeNil()) - g.Expect(*collisionCount).Should(BeEquivalentTo(0)) - g.Expect(len(revisionList)).Should(BeEquivalentTo(1)) - g.Expect(len(currentRevision.Name)).ShouldNot(BeEquivalentTo(0)) - g.Expect(updatedRevision.Name).Should(BeEquivalentTo(currentRevision.Name)) - v1RevisionName := updatedRevision.Name - waitingCacheUpdate(deploy.Namespace, 1) - adapter.currentRevision = updatedRevision.Name - - // updating deploy spec should construct a new updated CcontrollerRevision - deploy.Spec.Template.Spec.Containers[0].Image = "nginx:v2" - currentRevision, updatedRevision, revisionList, collisionCount, createNewRevision, err = revisionManager.ConstructRevisions(deploy, false) - g.Expect(err).ShouldNot(HaveOccurred()) - g.Expect(createNewRevision).Should(BeTrue()) - g.Expect(collisionCount).ShouldNot(BeNil()) - g.Expect(*collisionCount).Should(BeEquivalentTo(0)) - g.Expect(len(revisionList)).Should(BeEquivalentTo(2)) - g.Expect(len(currentRevision.Name)).ShouldNot(BeEquivalentTo(0)) - g.Expect(len(currentRevision.Name)).ShouldNot(BeEquivalentTo(0)) - g.Expect(updatedRevision.Name).ShouldNot(BeEquivalentTo(currentRevision.Name)) - waitingCacheUpdate(deploy.Namespace, 2) - - // reconcile with same spec with current revision is updated to updated revision - adapter.currentRevision = updatedRevision.Name - currentRevision, updatedRevision, revisionList, collisionCount, createNewRevision, err = revisionManager.ConstructRevisions(deploy, false) - g.Expect(err).Should(Succeed()) - g.Expect(createNewRevision).Should(BeFalse()) - g.Expect(collisionCount).ShouldNot(BeNil()) - g.Expect(*collisionCount).Should(BeEquivalentTo(0)) - g.Expect(len(revisionList)).Should(BeEquivalentTo(2)) - g.Expect(len(currentRevision.Name)).ShouldNot(BeEquivalentTo(0)) - g.Expect(len(currentRevision.Name)).ShouldNot(BeEquivalentTo(0)) - g.Expect(updatedRevision.Name).Should(BeEquivalentTo(currentRevision.Name)) - waitingCacheUpdate(deploy.Namespace, 2) - - // updating deploy spec to old version should not construct a new updated CcontrollerRevision - deploy.Spec.Template.Spec.Containers[0].Image = "nginx:v1" - currentRevision, updatedRevision, revisionList, collisionCount, createNewRevision, err = revisionManager.ConstructRevisions(deploy, false) - g.Expect(err).ShouldNot(HaveOccurred()) - g.Expect(createNewRevision).Should(BeFalse()) - g.Expect(collisionCount).ShouldNot(BeNil()) - g.Expect(*collisionCount).Should(BeEquivalentTo(0)) - g.Expect(len(revisionList)).Should(BeEquivalentTo(2)) - g.Expect(len(currentRevision.Name)).ShouldNot(BeEquivalentTo(0)) - g.Expect(len(currentRevision.Name)).ShouldNot(BeEquivalentTo(0)) - g.Expect(updatedRevision.Name).ShouldNot(BeEquivalentTo(currentRevision.Name)) - g.Expect(updatedRevision.Name).Should(BeEquivalentTo(v1RevisionName)) -} - -func TestRevisionCleanUp(t *testing.T) { - g = NewGomegaWithT(t) - testcase := "test-revision-cleanup" - schema := runtime.NewScheme() - corev1.AddToScheme(schema) - appsv1.AddToScheme(schema) - c = fake.NewClientBuilder().WithScheme(schema).Build() - g.Expect(createNamespace(c, testcase)).Should(Succeed()) - - deploy := &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: testcase, - Name: testcase, - }, - Spec: appsv1.DeploymentSpec{ - Selector: selector, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: selectedLabels, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "test", - Image: "nginx:v1", - }, - }, - }, - }, - }, - } - g.Expect(c.Create(context.TODO(), deploy)).Should(Succeed()) - - adapter := &OwnerAdapterImpl{ - name: testcase, - selector: selector, - selectedLabels: selectedLabels, - collisionCount: 0, - historyLimit: 0, // we at lease reserve 2 extra controller revisions - currentRevision: "test", - inUsed: true, - } - - revisionManager := NewRevisionManager(c, schema, adapter) - revisionManager.ConstructRevisions(deploy, false) - waitingCacheUpdate(deploy.Namespace, 1) - - deploy.Spec.Template.Spec.Containers[0].Image = "nginx:v2" - revisionManager.ConstructRevisions(deploy, false) - waitingCacheUpdate(deploy.Namespace, 2) - - deploy.Spec.Template.Spec.Containers[0].Image = "nginx:v3" - revisionManager.ConstructRevisions(deploy, false) - waitingCacheUpdate(deploy.Namespace, 3) - - revisionManager.ConstructRevisions(deploy, false) - waitingCacheUpdate(deploy.Namespace, 3) - - adapter.inUsed = false - revisionManager.ConstructRevisions(deploy, false) - waitingCacheUpdate(deploy.Namespace, 2) -} - -func TestRevisionCreation(t *testing.T) { - g = NewGomegaWithT(t) - testcase := "test-revision-creation" - schema := runtime.NewScheme() - corev1.AddToScheme(schema) - appsv1.AddToScheme(schema) - c = fake.NewClientBuilder().WithScheme(schema).Build() - g.Expect(createNamespace(c, testcase)).Should(Succeed()) - - deploy := &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: testcase, - Name: testcase, - }, - Spec: appsv1.DeploymentSpec{ - Selector: selector, - Template: corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: selectedLabels, - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "test", - Image: "nginx:v1", - }, - }, - }, - }, - }, - } - g.Expect(c.Create(context.TODO(), deploy)).Should(Succeed()) - - adapter := &OwnerAdapterImpl{ - name: testcase, - selector: selector, - selectedLabels: selectedLabels, - collisionCount: 0, - historyLimit: 0, // we at lease reserve 2 extra controller revisions - currentRevision: "test", - inUsed: true, - } - - revisionManager := NewRevisionManager(c, schema, adapter) - currentRevision, _, _, _, _, err := revisionManager.ConstructRevisions(deploy, false) - g.Expect(err).ShouldNot(HaveOccurred()) - waitingCacheUpdate(deploy.Namespace, 1) - - _, err = revisionManager.createControllerRevision(context.TODO(), deploy, currentRevision, nil) - g.Expect(err).Should(HaveOccurred()) - - var collisionCount int32 = 0 - // if new revision conflict with existing revision and their contents are equals, then will reuse the existing one - newRevision, err := revisionManager.newRevision(deploy, currentRevision.Revision+1, &collisionCount) - g.Expect(err).ShouldNot(HaveOccurred()) - newRevision, err = revisionManager.createControllerRevision(context.TODO(), deploy, newRevision, &collisionCount) - g.Expect(err).ShouldNot(HaveOccurred()) - g.Expect(newRevision.Name).Should(BeEquivalentTo(currentRevision.Name)) - - // change the data of existing revision - deployClone := deploy.DeepCopy() - deployClone.Spec.Template.Labels["foo"] = "foo" - currentRevision.Data.Raw, _ = revisionManager.ownerGetter.GetPatch(deployClone) - g.Expect(c.Update(context.TODO(), currentRevision)).Should(Succeed()) - // if their contents are not equals, it should regenerate a new name - newRevision, err = revisionManager.newRevision(deploy, currentRevision.Revision+1, &collisionCount) - g.Expect(err).ShouldNot(HaveOccurred()) - newRevision, err = revisionManager.createControllerRevision(context.TODO(), deploy, newRevision, &collisionCount) - g.Expect(err).ShouldNot(HaveOccurred()) - g.Expect(newRevision.Name).ShouldNot(BeEquivalentTo(currentRevision.Name)) -} - -func waitingCacheUpdate(namespace string, expectedRevisionCount int) { - g.Eventually(func() error { - revisionList := &appsv1.ControllerRevisionList{} - if err := c.List(context.TODO(), revisionList, &client.ListOptions{Namespace: namespace}); err != nil { - return err - } - - if len(revisionList.Items) != expectedRevisionCount { - return fmt.Errorf("expected %d, got %d\n", expectedRevisionCount, len(revisionList.Items)) - } - - return nil - }, 5*time.Second, 1*time.Second).Should(Succeed()) -} - -type OwnerAdapterImpl struct { - name string - selector *metav1.LabelSelector - selectedLabels map[string]string - collisionCount int32 - historyLimit int32 - currentRevision string - inUsed bool -} - -func (a OwnerAdapterImpl) GetSelector(obj metav1.Object) *metav1.LabelSelector { - return a.selector -} - -func (a OwnerAdapterImpl) GetCollisionCount(obj metav1.Object) *int32 { - return &a.collisionCount -} - -func (a OwnerAdapterImpl) GetHistoryLimit(obj metav1.Object) int32 { - return a.historyLimit -} - -func (a OwnerAdapterImpl) GetPatch(obj metav1.Object) ([]byte, error) { - // mock patch - return json.Marshal(obj) -} - -func (a OwnerAdapterImpl) GetCurrentRevision(obj metav1.Object) string { - return a.currentRevision -} - -func (a OwnerAdapterImpl) IsInUsed(obj metav1.Object, controllerRevision string) bool { - return a.inUsed -} - -func createNamespace(c client.Client, namespaceName string) error { - ns := &corev1.Namespace{ - ObjectMeta: metav1.ObjectMeta{ - Name: namespaceName, - }, - } - - return c.Create(context.TODO(), ns) -} diff --git a/controller/utils/error.go b/controller/utils/error.go deleted file mode 100644 index 70a2648d..00000000 --- a/controller/utils/error.go +++ /dev/null @@ -1,40 +0,0 @@ -/* -Copyright 2024 The KusionStack Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package utils - -import "fmt" - -func AggregateErrors(errs []error) error { - if len(errs) == 0 { - return nil - } else if len(errs) == 1 { - return errs[0] - } - - var aggErr error - for _, currErr := range errs { - if currErr == nil { - continue - } - if aggErr == nil { - aggErr = currErr - continue - } - aggErr = fmt.Errorf("%w; %w", aggErr, currErr) - } - return aggErr -} diff --git a/controller/utils/error_test.go b/controller/utils/error_test.go deleted file mode 100644 index 7ec53e96..00000000 --- a/controller/utils/error_test.go +++ /dev/null @@ -1,54 +0,0 @@ -/* -Copyright 2024 The KusionStack Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package utils - -import ( - "fmt" - "testing" -) - -func TestError(t *testing.T) { - var errs []error - err1 := fmt.Errorf("error 1") - err2 := fmt.Errorf("error 2") - var err3 error - - actual := AggregateErrors(errs) - if actual != nil { - t.Fatalf("expect %v equal to nil", actual) - } - - errs = append(errs, err1) - actual = AggregateErrors(errs) - if actual.Error() != err1.Error() { - t.Fatalf("expect %v equal to %v", actual, err1) - } - - errs = append(errs, err2) - actual = AggregateErrors(errs) - expected := fmt.Errorf("%w; %w", errs[0], errs[1]) - if actual.Error() != expected.Error() { - t.Fatalf("expect %v equal to %v", actual, expected) - } - - errs = append(errs, err3) - actual = AggregateErrors(errs) - expected = fmt.Errorf("%w; %w", errs[0], errs[1]) - if actual.Error() != expected.Error() { - t.Fatalf("expect %v equal to %v", actual, expected) - } -} diff --git a/controller/utils/finalizer.go b/controller/utils/finalizer.go deleted file mode 100644 index d69c9732..00000000 --- a/controller/utils/finalizer.go +++ /dev/null @@ -1,64 +0,0 @@ -/* - Copyright 2023 The KusionStack Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package utils - -import ( - "context" - - "k8s.io/apimachinery/pkg/types" - "k8s.io/client-go/util/retry" - "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" -) - -func RemoveFinalizer(ctx context.Context, c client.Client, obj client.Object, finalizer string) error { - return retry.RetryOnConflict(retry.DefaultRetry, func() error { - controllerutil.RemoveFinalizer(obj, finalizer) - var updateErr error - if updateErr = c.Update(ctx, obj); updateErr == nil { - return nil - } - if err := c.Get(ctx, types.NamespacedName{Namespace: obj.GetNamespace(), Name: obj.GetName()}, obj); err != nil { - return err - } - return updateErr - }) -} - -func AddFinalizer(ctx context.Context, c client.Client, obj client.Object, finalizer string) error { - return retry.RetryOnConflict(retry.DefaultRetry, func() error { - controllerutil.AddFinalizer(obj, finalizer) - var updateErr error - if updateErr = c.Update(ctx, obj); updateErr == nil { - return nil - } - if err := c.Get(ctx, types.NamespacedName{Namespace: obj.GetNamespace(), Name: obj.GetName()}, obj); err != nil { - return err - } - return updateErr - }) -} - -func ContainsFinalizer(obj client.Object, finalizer string) bool { - for _, f := range obj.GetFinalizers() { - if f == finalizer { - return true - } - } - - return false -} diff --git a/xset/revisionadapter/revision_adapter.go b/xset/revisionowner/revision_owner.go similarity index 53% rename from xset/revisionadapter/revision_adapter.go rename to xset/revisionowner/revision_owner.go index ec84afd3..1216dccc 100644 --- a/xset/revisionadapter/revision_adapter.go +++ b/xset/revisionowner/revision_owner.go @@ -14,7 +14,7 @@ * limitations under the License. */ -package revisionadapter +package revisionowner import ( "context" @@ -22,71 +22,93 @@ import ( appsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/sets" + "kusionstack.io/kube-api/apps/v1alpha1" + "kusionstack.io/kube-utils/controller/history" "kusionstack.io/kube-utils/xset/api" "kusionstack.io/kube-utils/xset/xcontrol" ) -type RevisionOwnerAdapter struct { +var _ history.RevisionOwner = &revisionOwner{} + +type revisionOwner struct { api.XSetController xcontrol.TargetControl } -func NewRevisionOwnerAdapter(xsetController api.XSetController, xcontrol xcontrol.TargetControl) *RevisionOwnerAdapter { - return &RevisionOwnerAdapter{ +func NewRevisionOwner(xsetController api.XSetController, xcontrol xcontrol.TargetControl) *revisionOwner { + return &revisionOwner{ XSetController: xsetController, TargetControl: xcontrol, } } -func (r *RevisionOwnerAdapter) GetSelector(obj metav1.Object) *metav1.LabelSelector { - xset := obj.(api.XSetObject) - return r.XSetController.GetXSetSpec(xset).Selector -} - -func (r *RevisionOwnerAdapter) GetCollisionCount(obj metav1.Object) *int32 { - xset := obj.(api.XSetObject) - return r.XSetController.GetXSetStatus(xset).CollisionCount -} - -func (r *RevisionOwnerAdapter) GetHistoryLimit(obj metav1.Object) int32 { - xset := obj.(api.XSetObject) - return r.XSetController.GetXSetSpec(xset).HistoryLimit -} - -func (r *RevisionOwnerAdapter) GetPatch(obj metav1.Object) ([]byte, error) { - return r.getXSetPatch(obj) +func (r *revisionOwner) GetGroupVersionKind() schema.GroupVersionKind { + meta := r.XSetController.XSetMeta() + return meta.GroupVersionKind() } -func (r *RevisionOwnerAdapter) GetCurrentRevision(obj metav1.Object) string { - xset := obj.(api.XSetObject) - return r.XSetController.GetXSetStatus(xset).CurrentRevision +func (r *revisionOwner) GetMatchLabels(parent metav1.Object) map[string]string { + obj := parent.(api.XSetObject) + xsetSpec := r.XSetController.GetXSetSpec(obj) + if len(xsetSpec.Selector.MatchLabels) > 0 { + return xsetSpec.Selector.MatchLabels + } else { + return map[string]string{ + v1alpha1.ControlledByKusionStackLabelKey: "xset", + "xset.kusionstack.io/name": obj.GetName(), + } + } } -func (r *RevisionOwnerAdapter) IsInUsed(obj metav1.Object, revision string) bool { - xSetObject, _ := obj.(api.XSetObject) +func (r *revisionOwner) GetInUsedRevisions(parent metav1.Object) (sets.String, error) { + xSetObject, _ := parent.(api.XSetObject) spec := r.XSetController.GetXSetSpec(xSetObject) status := r.XSetController.GetXSetStatus(xSetObject) - if status.UpdatedRevision == revision || status.CurrentRevision == revision { - return true - } + res := sets.String{} - targets, _ := r.TargetControl.GetFilteredTargets(context.TODO(), spec.Selector, xSetObject) + res.Insert(status.UpdatedRevision) + res.Insert(status.CurrentRevision) + + targets, err := r.TargetControl.GetFilteredTargets(context.TODO(), spec.Selector, xSetObject) + if err != nil { + return nil, err + } for _, target := range targets { if target.GetLabels() != nil { currentRevisionName, exist := target.GetLabels()[appsv1.ControllerRevisionHashLabelKey] - if exist && currentRevisionName == revision { - return true + if exist { + res.Insert(currentRevisionName) } } } + return res, nil +} + +func (r *revisionOwner) GetCollisionCount(obj metav1.Object) *int32 { + xset := obj.(api.XSetObject) + return r.XSetController.GetXSetStatus(xset).CollisionCount +} - return false +func (r *revisionOwner) GetHistoryLimit(obj metav1.Object) int32 { + xset := obj.(api.XSetObject) + return r.XSetController.GetXSetSpec(xset).HistoryLimit +} + +func (r *revisionOwner) GetPatch(obj metav1.Object) ([]byte, error) { + return r.getXSetPatch(obj) +} + +func (r *revisionOwner) GetCurrentRevision(obj metav1.Object) string { + xset := obj.(api.XSetObject) + return r.XSetController.GetXSetStatus(xset).CurrentRevision } -func (r *RevisionOwnerAdapter) getXSetPatch(obj metav1.Object) ([]byte, error) { +func (r *revisionOwner) getXSetPatch(obj metav1.Object) ([]byte, error) { xset := obj.(api.XSetObject) xTpl := r.XSetController.GetXTemplate(xset) tplBytes, err := json.Marshal(xTpl) diff --git a/xset/synccontrols/sync_control.go b/xset/synccontrols/sync_control.go index d22e7d3d..98876d83 100644 --- a/xset/synccontrols/sync_control.go +++ b/xset/synccontrols/sync_control.go @@ -18,6 +18,7 @@ package synccontrols import ( "context" + "errors" "fmt" "strconv" "strings" @@ -26,7 +27,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" @@ -292,7 +293,7 @@ func (r *RealSyncControl) dealIncludeExcludeTargets(ctx context.Context, xsetObj if notAllowedIncludeTargets.Len() > 0 { r.Recorder.Eventf(xsetObject, corev1.EventTypeWarning, "IncludeNotAllowed", fmt.Sprintf("targets [%v] are not allowed to include, please find out the reason from target's event", notAllowedIncludeTargets.List())) } - return toExcludeTargets, toIncludeTargets, controllerutils.AggregateErrors([]error{exErr, inErr}) + return toExcludeTargets, toIncludeTargets, errors.Join(exErr, inErr) } // checkAllowFunc refers to AllowResourceExclude and AllowResourceInclude @@ -306,7 +307,7 @@ func (r *RealSyncControl) allowIncludeExcludeTargets(ctx context.Context, xset a target := r.xsetController.EmptyXObject() targetName := targetNames[i] err = r.Client.Get(ctx, types.NamespacedName{Namespace: xset.GetNamespace(), Name: targetName}, target) - if errors.IsNotFound(err) { + if apierrors.IsNotFound(err) { notAllowTargets.Insert(targetNames[i]) continue } else if err != nil { @@ -460,7 +461,7 @@ func (r *RealSyncControl) Scale(ctx context.Context, xsetObject api.XSetObject, if updateContextErr := retry.RetryOnConflict(retry.DefaultRetry, func() error { return resourcecontexts.UpdateToTargetContext(r.xsetController, r.Client, r.cacheExpectation, xsetObject, syncContext.OwnedIds) }); updateContextErr != nil { - err = controllerutils.AggregateErrors([]error{updateContextErr, err}) + err = errors.Join(updateContextErr, err) } } if err != nil { diff --git a/xset/synccontrols/x_scale.go b/xset/synccontrols/x_scale.go index 35a5d854..66ee222d 100644 --- a/xset/synccontrols/x_scale.go +++ b/xset/synccontrols/x_scale.go @@ -18,6 +18,7 @@ package synccontrols import ( "context" + "errors" "sort" "strconv" @@ -120,7 +121,7 @@ func (r *RealSyncControl) doIncludeExcludeTargets(ctx context.Context, xset api. defer func() { includeErrs = append(includeErrs, err) }() return r.includeTarget(ctx, xset, includeTargets[idx], strconv.Itoa(availableContexts[idx].ID)) }) - return controllerutils.AggregateErrors(append(includeErrs, excludeErrs...)) + return errors.Join(append(includeErrs, excludeErrs...)...) } // excludeTarget try to exclude a target from xset diff --git a/xset/xcontrol/target_control.go b/xset/xcontrol/target_control.go index f9ba197c..e06d37f3 100644 --- a/xset/xcontrol/target_control.go +++ b/xset/xcontrol/target_control.go @@ -18,6 +18,7 @@ package xcontrol import ( "context" + "errors" "fmt" "reflect" @@ -31,7 +32,6 @@ import ( "kusionstack.io/kube-utils/controller/expectations" "kusionstack.io/kube-utils/controller/mixin" refmanagerutil "kusionstack.io/kube-utils/controller/refmanager" - "kusionstack.io/kube-utils/controller/utils" "kusionstack.io/kube-utils/xset/api" ) @@ -200,7 +200,7 @@ func (r *targetControl) getTargets(candidates []client.Object, selector *metav1. } } - return claimObjs, utils.AggregateErrors(errList) + return claimObjs, errors.Join(errList...) } func setUpCache(cache cache.Cache, controller api.XSetController) error { diff --git a/xset/xset_controller.go b/xset/xset_controller.go index 7de410b6..d19cdb9e 100644 --- a/xset/xset_controller.go +++ b/xset/xset_controller.go @@ -1,11 +1,28 @@ +/* + * Copyright 2024-2025 KusionStack Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package xset import ( "context" + "errors" "fmt" "time" - "k8s.io/apimachinery/pkg/api/errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/clock" @@ -16,13 +33,13 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" + clientutil "kusionstack.io/kube-utils/client" "kusionstack.io/kube-utils/controller/expectations" + "kusionstack.io/kube-utils/controller/history" "kusionstack.io/kube-utils/controller/mixin" - "kusionstack.io/kube-utils/controller/revision" - controllerutils "kusionstack.io/kube-utils/controller/utils" "kusionstack.io/kube-utils/xset/api" "kusionstack.io/kube-utils/xset/resourcecontexts" - "kusionstack.io/kube-utils/xset/revisionadapter" + "kusionstack.io/kube-utils/xset/revisionowner" "kusionstack.io/kube-utils/xset/synccontrols" "kusionstack.io/kube-utils/xset/xcontrol" ) @@ -39,7 +56,7 @@ type xSetCommonReconciler struct { cacheExpectation *expectations.CacheExpectation targetControl xcontrol.TargetControl syncControl synccontrols.SyncControl - revisionManager *revision.RevisionManager + revisionManager history.HistoryManager } func SetUpWithManager(mgr ctrl.Manager, xsetController api.XSetController) error { @@ -56,7 +73,10 @@ func SetUpWithManager(mgr ctrl.Manager, xsetController api.XSetController) error targetControl, err := xcontrol.NewTargetControl(reconcilerMixin, xsetController, cacheExpectation) syncControl := synccontrols.NewRealSyncControl(reconcilerMixin, xsetController, targetControl, cacheExpectation) - revisionManager := revision.NewRevisionManager(reconcilerMixin.Client, reconcilerMixin.Scheme, revisionadapter.NewRevisionOwnerAdapter(xsetController, targetControl)) + revisionControl := history.NewRevisionControl(reconcilerMixin.Client, reconcilerMixin.Client) + revisionOwner := revisionowner.NewRevisionOwner(xsetController, targetControl) + revisionManager := history.NewHistoryManager(revisionControl, revisionOwner) + reconciler := &xSetCommonReconciler{ targetControl: targetControl, ReconcilerMixin: *reconcilerMixin, @@ -97,7 +117,7 @@ func (r *xSetCommonReconciler) Reconcile(ctx context.Context, req reconcile.Requ logger := r.Logger.WithValues(kind, key) instance := r.XSetController.EmptyXSetObject() if err := r.Client.Get(ctx, req.NamespacedName, instance); err != nil { - if !errors.IsNotFound(err) { + if !apierrors.IsNotFound(err) { logger.Error(err, "failed to find object") return reconcile.Result{}, err } @@ -113,11 +133,6 @@ func (r *xSetCommonReconciler) Reconcile(ctx context.Context, req reconcile.Requ } if instance.GetDeletionTimestamp() != nil { - // todo, why reclaim ownerReferences before target deletion? - if err := r.ensureReclaimTargetOwnerReferences(ctx, instance); err != nil { - // reclaim targets ownerReferences before remove finalizers - return ctrl.Result{}, err - } if err := r.ensureReclaimTargetsDeletion(ctx, instance); err != nil { // reclaim targets deletion before remove finalizers return ctrl.Result{}, err @@ -127,7 +142,7 @@ func (r *xSetCommonReconciler) Reconcile(ctx context.Context, req reconcile.Requ if err := resourcecontexts.UpdateToTargetContext(r.XSetController, r.Client, r.cacheExpectation, instance, nil); err != nil { return ctrl.Result{}, err } - if err := controllerutils.RemoveFinalizer(ctx, r.Client, instance, r.finalizerName); err != nil { + if err := clientutil.RemoveFinalizerAndUpdate(ctx, r.Client, instance, r.finalizerName); err != nil { return ctrl.Result{}, err } } @@ -136,10 +151,10 @@ func (r *xSetCommonReconciler) Reconcile(ctx context.Context, req reconcile.Requ } if !controllerutil.ContainsFinalizer(instance, r.finalizerName) { - return ctrl.Result{}, controllerutils.AddFinalizer(context.TODO(), r.Client, instance, r.finalizerName) + return ctrl.Result{}, clientutil.AddFinalizerAndUpdate(ctx, r.Client, instance, r.finalizerName) } - currentRevision, updatedRevision, revisions, collisionCount, _, err := r.revisionManager.ConstructRevisions(instance, false) + currentRevision, updatedRevision, revisions, collisionCount, _, err := r.revisionManager.ConstructRevisions(ctx, instance) if err != nil { return ctrl.Result{}, fmt.Errorf("fail to construct revision for %s %s: %s", kind, key, err.Error()) } @@ -148,7 +163,7 @@ func (r *xSetCommonReconciler) Reconcile(ctx context.Context, req reconcile.Requ newStatus := xsetStatus.DeepCopy() newStatus.UpdatedRevision = updatedRevision.Name newStatus.CurrentRevision = currentRevision.Name - newStatus.CollisionCount = collisionCount + newStatus.CollisionCount = &collisionCount syncContext := &synccontrols.SyncContext{ Revisions: revisions, CurrentRevision: currentRevision, @@ -183,39 +198,13 @@ func (r *xSetCommonReconciler) doSync(ctx context.Context, instance api.XSetObje _, scaleRequeueAfter, scaleErr := r.syncControl.Scale(ctx, instance, syncContext) _, updateRequeueAfter, updateErr := r.syncControl.Update(ctx, instance, syncContext) - err = controllerutils.AggregateErrors([]error{scaleErr, updateErr}) + err = errors.Join(scaleErr, updateErr) if updateRequeueAfter != nil && (scaleRequeueAfter == nil || *updateRequeueAfter < *scaleRequeueAfter) { return updateRequeueAfter, err } return scaleRequeueAfter, err } -func (r *xSetCommonReconciler) ensureReclaimTargetOwnerReferences(ctx context.Context, instance api.XSetObject) error { - xspec := r.XSetController.GetXSetSpec(instance) - targets, err := r.targetControl.GetFilteredTargets(ctx, xspec.Selector, instance) - if err != nil { - return fmt.Errorf("fail to get filtered Targets: %s", err.Error()) - } - for i := range targets { - target := targets[i] - targetOwner := target.GetOwnerReferences() - if len(targetOwner) == 0 { - continue - } - var newOwnerRefs []metav1.OwnerReference - for j := range targetOwner { - newOwnerRefs = append(newOwnerRefs, targetOwner[j]) - } - if len(newOwnerRefs) != len(targetOwner) { - target.SetOwnerReferences(newOwnerRefs) - if err := r.targetControl.UpdateTarget(ctx, target); err != nil { - return err - } - } - } - return nil -} - func (r *xSetCommonReconciler) ensureReclaimTargetsDeletion(ctx context.Context, instance api.XSetObject) error { xspec := r.XSetController.GetXSetSpec(instance) targets, err := r.targetControl.GetFilteredTargets(ctx, xspec.Selector, instance) From 905d851c076846c0ce102386692807c496c65cf0 Mon Sep 17 00:00:00 2001 From: hexin <574252631@qq.com> Date: Mon, 21 Jul 2025 16:37:19 +0800 Subject: [PATCH 3/5] Xset some optimization (#77) * update comment * add GetXSetPatch for controller * add availableReplicas updatedReadyReplicas ScheduledReplicas * remove GetXTemplate, add GetXSetPatch GetXObjectFromRevision * const to var --- controller/utils/slow_start.go | 2 +- controller/utils/slow_start_test.go | 2 +- xset/api/xset_controller_types.go | 7 ++++++- xset/api/xset_types.go | 14 ++++++++++++++ xset/opslifecycle/const.go | 2 +- xset/opslifecycle/utils.go | 2 +- xset/resourcecontexts/resource_context.go | 2 +- xset/revisionowner/revision_owner.go | 22 +--------------------- xset/synccontrols/const.go | 4 ++-- xset/synccontrols/sync_control.go | 18 ++++++++++++++++-- xset/synccontrols/x_replace.go | 2 +- xset/synccontrols/x_scale.go | 2 +- xset/synccontrols/x_update.go | 2 +- xset/synccontrols/x_utils.go | 5 +---- 14 files changed, 48 insertions(+), 38 deletions(-) diff --git a/controller/utils/slow_start.go b/controller/utils/slow_start.go index 5a506689..5d8697dc 100644 --- a/controller/utils/slow_start.go +++ b/controller/utils/slow_start.go @@ -1,6 +1,6 @@ /* Copyright 2016 The Kubernetes Authors. -Copyright 2023 The KusionStack Authors. +Copyright 2024-2025 The KusionStack Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/controller/utils/slow_start_test.go b/controller/utils/slow_start_test.go index 95adebc8..50a9356f 100644 --- a/controller/utils/slow_start_test.go +++ b/controller/utils/slow_start_test.go @@ -1,5 +1,5 @@ /* -Copyright 2023 The KusionStack Authors. +Copyright 2024-2025 The KusionStack Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/xset/api/xset_controller_types.go b/xset/api/xset_controller_types.go index 7c9cdca4..758ff25a 100644 --- a/xset/api/xset_controller_types.go +++ b/xset/api/xset_controller_types.go @@ -13,9 +13,11 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package api import ( + appsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -29,8 +31,10 @@ type XSetController interface { EmptyXSetObject() XSetObject EmptyXObject() client.Object EmptyXObjectList() client.ObjectList + GetXObjectFromRevision(revision *appsv1.ControllerRevision) (client.Object, error) GetXSetSpec(object XSetObject) *XSetSpec + GetXSetPatch(object metav1.Object) ([]byte, error) UpdateScaleStrategy(object XSetObject, scaleStrategy *ScaleStrategy) (err error) GetXSetStatus(object XSetObject) *XSetStatus SetXSetStatus(object XSetObject, status *XSetStatus) @@ -38,8 +42,9 @@ type XSetController interface { GetScaleInOpsLifecycleAdapter() LifecycleAdapter GetUpdateOpsLifecycleAdapter() LifecycleAdapter - GetXTemplate(object XSetObject) client.Object + CheckScheduled(object client.Object) bool CheckReady(object client.Object) bool + CheckAvailable(object client.Object) bool } type XSetObject client.Object diff --git a/xset/api/xset_types.go b/xset/api/xset_types.go index 0f78c27e..ff8056d8 100644 --- a/xset/api/xset_types.go +++ b/xset/api/xset_types.go @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package api // +k8s:deepcopy-gen=file @@ -166,6 +167,10 @@ type XSetStatus struct { // +optional Replicas int32 `json:"replicas,omitempty"` + // the number of scheduled replicas for the replicas set. + // +optional + ScheduledReplicas int32 `json:"scheduledReplicas,omitempty"` + // The number of targets in updated version. // +optional UpdatedReplicas int32 `json:"updatedReplicas,omitempty"` @@ -178,6 +183,15 @@ type XSetStatus struct { // +optional UpdatedReadyReplicas int32 `json:"updatedReadyReplicas,omitempty"` + // The number of available replicas for this replica set. + // +optional + AvailableReplicas int32 `json:"availableReplicas,omitempty"` + + // UpdatedAvailableReplicas indicates the number of available updated revision replicas for this replicas set. + // A model is updated available means the model is ready for updated revision and accessible + // +optional + UpdatedAvailableReplicas int32 `json:"updatedAvailableReplicas,omitempty"` + // Represents the latest available observations of a XSet's current state. // +optional Conditions []metav1.Condition `json:"conditions,omitempty"` diff --git a/xset/opslifecycle/const.go b/xset/opslifecycle/const.go index 4edd9614..27b876f5 100644 --- a/xset/opslifecycle/const.go +++ b/xset/opslifecycle/const.go @@ -17,7 +17,7 @@ package opslifecycle // TargetOpsLifecycle labels -const ( +var ( // TargetOperatingLabelPrefix indicates a Target is operating TargetOperatingLabelPrefix = "operating.opslifecycle.kusionstack.io" // TargetOperationTypeLabelPrefix indicates the type of operation diff --git a/xset/opslifecycle/utils.go b/xset/opslifecycle/utils.go index 71323261..cfbe475f 100644 --- a/xset/opslifecycle/utils.go +++ b/xset/opslifecycle/utils.go @@ -1,5 +1,5 @@ /* - Copyright 2023-2025 The KusionStack Authors. + Copyright 2024-2025 The KusionStack Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/xset/resourcecontexts/resource_context.go b/xset/resourcecontexts/resource_context.go index d234f14b..ef4debde 100644 --- a/xset/resourcecontexts/resource_context.go +++ b/xset/resourcecontexts/resource_context.go @@ -1,5 +1,5 @@ /* -Copyright 2023 The KusionStack Authors. +Copyright 2023-2025 The KusionStack Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/xset/revisionowner/revision_owner.go b/xset/revisionowner/revision_owner.go index 1216dccc..ba0fd173 100644 --- a/xset/revisionowner/revision_owner.go +++ b/xset/revisionowner/revision_owner.go @@ -18,7 +18,6 @@ package revisionowner import ( "context" - "encoding/json" appsv1 "k8s.io/api/apps/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -109,24 +108,5 @@ func (r *revisionOwner) GetCurrentRevision(obj metav1.Object) string { } func (r *revisionOwner) getXSetPatch(obj metav1.Object) ([]byte, error) { - xset := obj.(api.XSetObject) - xTpl := r.XSetController.GetXTemplate(xset) - tplBytes, err := json.Marshal(xTpl) - if err != nil { - return nil, err - } - tplMap := make(map[string]interface{}) - err = json.Unmarshal(tplBytes, &tplMap) - if err != nil { - return nil, err - } - - tplMap["$patch"] = "replace" - specMap := make(map[string]interface{}) - specMap["template"] = tplMap - objMap := make(map[string]interface{}) - objMap["spec"] = specMap - - patch, err := json.Marshal(objMap) - return patch, err + return r.XSetController.GetXSetPatch(obj) } diff --git a/xset/synccontrols/const.go b/xset/synccontrols/const.go index 70edd4b4..0d70e3ea 100644 --- a/xset/synccontrols/const.go +++ b/xset/synccontrols/const.go @@ -17,7 +17,7 @@ package synccontrols // XSet labels -const ( +var ( // TargetInstanceIDLabelKey is used to attach target instance ID on target TargetInstanceIDLabelKey = "xset.kusionstack.io/instance-id" // XSetUpdateIndicateLabelKey is used to indicate a target should be updated by label @@ -41,6 +41,6 @@ const ( TargetOrphanedIndicateLabelKey = "xset.kusionstack.io/orphaned" ) -const ( +var ( LastTargetStatusAnnotationKey = "xset.kusionstack.io/last-target-status" ) diff --git a/xset/synccontrols/sync_control.go b/xset/synccontrols/sync_control.go index 98876d83..4e2a04bd 100644 --- a/xset/synccontrols/sync_control.go +++ b/xset/synccontrols/sync_control.go @@ -1,5 +1,5 @@ /* -Copyright 2023 The KusionStack Authors. +Copyright 2023-2025 The KusionStack Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -744,7 +744,7 @@ func (r *RealSyncControl) CalculateStatus(ctx context.Context, instance api.XSet newStatus := syncContext.NewStatus newStatus.ObservedGeneration = instance.GetGeneration() - var readyReplicas, replicas, updatedReplicas, operatingReplicas, updatedReadyReplicas int32 + var readyReplicas, scheduledReplicas, replicas, updatedReplicas, operatingReplicas, updatedReadyReplicas, availableReplicas, updatedAvailableReplicas int32 activeTargets := FilterOutActiveTargetWrappers(syncContext.TargetWrappers) for _, targetWrapper := range activeTargets { @@ -770,6 +770,17 @@ func (r *RealSyncControl) CalculateStatus(ctx context.Context, instance api.XSet updatedReadyReplicas++ } } + + if r.xsetController.CheckAvailable(targetWrapper.Object) { + availableReplicas++ + if isUpdated { + updatedAvailableReplicas++ + } + } + + if r.xsetController.CheckScheduled(targetWrapper.Object) { + scheduledReplicas++ + } } newStatus.ReadyReplicas = readyReplicas @@ -777,6 +788,9 @@ func (r *RealSyncControl) CalculateStatus(ctx context.Context, instance api.XSet newStatus.UpdatedReplicas = updatedReplicas newStatus.OperatingReplicas = operatingReplicas newStatus.UpdatedReadyReplicas = updatedReadyReplicas + newStatus.ScheduledReplicas = scheduledReplicas + newStatus.AvailableReplicas = availableReplicas + newStatus.UpdatedAvailableReplicas = updatedAvailableReplicas spec := r.xsetController.GetXSetSpec(instance) if (spec.Replicas == nil && newStatus.UpdatedReadyReplicas >= 0) || diff --git a/xset/synccontrols/x_replace.go b/xset/synccontrols/x_replace.go index 8c5ab468..a5a58b50 100644 --- a/xset/synccontrols/x_replace.go +++ b/xset/synccontrols/x_replace.go @@ -1,5 +1,5 @@ /* -Copyright 2024 The KusionStack Authors. +Copyright 2024-2025 The KusionStack Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/xset/synccontrols/x_scale.go b/xset/synccontrols/x_scale.go index 66ee222d..0b3a68eb 100644 --- a/xset/synccontrols/x_scale.go +++ b/xset/synccontrols/x_scale.go @@ -1,5 +1,5 @@ /* -Copyright 2023 The KusionStack Authors. +Copyright 2024-2025 The KusionStack Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/xset/synccontrols/x_update.go b/xset/synccontrols/x_update.go index 8256de60..05f96a78 100644 --- a/xset/synccontrols/x_update.go +++ b/xset/synccontrols/x_update.go @@ -1,5 +1,5 @@ /* -Copyright 2023 The KusionStack Authors. +Copyright 2024-2025 The KusionStack Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/xset/synccontrols/x_utils.go b/xset/synccontrols/x_utils.go index d9327518..1cad1f63 100644 --- a/xset/synccontrols/x_utils.go +++ b/xset/synccontrols/x_utils.go @@ -17,7 +17,6 @@ package synccontrols import ( - "encoding/json" "fmt" "strconv" "time" @@ -51,15 +50,13 @@ func GetInstanceID(target client.Object) (int, error) { } func NewTargetFrom(setController api.XSetController, owner api.XSetObject, revision *appsv1.ControllerRevision, id int, updateFuncs ...func(client.Object) error) (client.Object, error) { - setObj := setController.EmptyXSetObject() - err := json.Unmarshal(revision.Data.Raw, setObj) + targetObj, err := setController.GetXObjectFromRevision(revision) if err != nil { return nil, err } meta := setController.XSetMeta() ownerRef := metav1.NewControllerRef(owner, meta.GroupVersionKind()) - targetObj := setController.GetXTemplate(setObj) targetObj.SetOwnerReferences(append(targetObj.GetOwnerReferences(), *ownerRef)) targetObj.SetNamespace(owner.GetNamespace()) targetObj.SetGenerateName(GetTargetsPrefix(owner.GetName())) From 049812eac7395c1476d3aa6e639fb141cc840bff Mon Sep 17 00:00:00 2001 From: AnnaYue Date: Tue, 22 Jul 2025 17:55:07 +0800 Subject: [PATCH 4/5] use lifecycle label manager to support dynamic label management (#79) --- xset/api/opslifecycle_types.go | 36 ++++- xset/api/xset_controller_types.go | 1 + xset/opslifecycle/const.go | 76 --------- ...lt_opslifecycle.go => default_adapters.go} | 12 +- xset/opslifecycle/label_manager.go | 72 +++++++++ xset/opslifecycle/utils.go | 153 +++++++++--------- xset/opslifecycle/utils_test.go | 23 +-- xset/synccontrols/const.go | 2 + xset/synccontrols/sync_control.go | 35 ++-- xset/synccontrols/x_replace.go | 7 +- xset/synccontrols/x_scale.go | 2 +- xset/synccontrols/x_update.go | 15 +- 12 files changed, 231 insertions(+), 203 deletions(-) delete mode 100644 xset/opslifecycle/const.go rename xset/opslifecycle/{default_opslifecycle.go => default_adapters.go} (87%) create mode 100644 xset/opslifecycle/label_manager.go diff --git a/xset/api/opslifecycle_types.go b/xset/api/opslifecycle_types.go index 494d12c4..78ac10cd 100644 --- a/xset/api/opslifecycle_types.go +++ b/xset/api/opslifecycle_types.go @@ -20,6 +20,40 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +type OperationLabelEnum int + +const ( + // OperatingLabelPrefix indicates a target is under operation + // set by xset controller + OperatingLabelPrefix OperationLabelEnum = iota + + // OperationTypeLabelPrefix indicates the type of operation + // set by xset controller + OperationTypeLabelPrefix + + // OperateLabelPrefix indicates a target could start operation + // set by related opsLifecycle controller. + // xset controller will start operation only after this label is set + OperateLabelPrefix + + // UndoOperationTypeLabelPrefix indicates a type of operation has been canceled. + // need to be handled by related opsLifecycle controller + UndoOperationTypeLabelPrefix + + // ServiceAvailableLabel indicates a target is available for service. + // set by related opsLifecycle controller. + ServiceAvailableLabel + + // PreparingDeleteLabel indicates a target is preparing to be deleted. + // set by xset controller, + // handle by related opsLifecycle controller if needed. + PreparingDeleteLabel +) + +type LifeCycleLabelManager interface { + Get(labelType OperationLabelEnum) string +} + type OperationType string var ( @@ -28,8 +62,6 @@ var ( OpsLifecycleTypeDelete OperationType = "delete" ) -type UpdateFunc func(object client.Object) (bool, error) - // LifecycleAdapter helps CRD Operators to easily access TargetOpsLifecycle type LifecycleAdapter interface { // GetID indicates ID of one TargetOpsLifecycle diff --git a/xset/api/xset_controller_types.go b/xset/api/xset_controller_types.go index 758ff25a..e8549fa2 100644 --- a/xset/api/xset_controller_types.go +++ b/xset/api/xset_controller_types.go @@ -39,6 +39,7 @@ type XSetController interface { GetXSetStatus(object XSetObject) *XSetStatus SetXSetStatus(object XSetObject, status *XSetStatus) + GetLifeCycleLabelManager() LifeCycleLabelManager GetScaleInOpsLifecycleAdapter() LifecycleAdapter GetUpdateOpsLifecycleAdapter() LifecycleAdapter diff --git a/xset/opslifecycle/const.go b/xset/opslifecycle/const.go deleted file mode 100644 index 27b876f5..00000000 --- a/xset/opslifecycle/const.go +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright 2024-2025 KusionStack Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package opslifecycle - -// TargetOpsLifecycle labels -var ( - // TargetOperatingLabelPrefix indicates a Target is operating - TargetOperatingLabelPrefix = "operating.opslifecycle.kusionstack.io" - // TargetOperationTypeLabelPrefix indicates the type of operation - TargetOperationTypeLabelPrefix = "operation-type.opslifecycle.kusionstack.io" - // TargetOperationPermissionLabelPrefix indicates the permission of operation - TargetOperationPermissionLabelPrefix = "operation-permission.opslifecycle.kusionstack.io" - // TargetUndoOperationTypeLabelPrefix indicates the type of operation has been canceled - TargetUndoOperationTypeLabelPrefix = "undo-operation-type.opslifecycle.kusionstack.io" - // TargetDoneOperationTypeLabelPrefix indicates the type of operation has been done - TargetDoneOperationTypeLabelPrefix = "done-operation-type.opslifecycle.kusionstack.io" - - // TargetPreCheckLabelPrefix indicates a Target is in pre-check phase - TargetPreCheckLabelPrefix = "pre-check.opslifecycle.kusionstack.io" - // TargetPreCheckedLabelPrefix indicates a Target has finished pre-check phase - TargetPreCheckedLabelPrefix = "pre-checked.opslifecycle.kusionstack.io" - // TargetPreparingLabelPrefix indicates a Target is preparing for operation - TargetPreparingLabelPrefix = "preparing.opslifecycle.kusionstack.io" - // TargetOperateLabelPrefix indicates a Target is in operate phase - TargetOperateLabelPrefix = "operate.opslifecycle.kusionstack.io" - // TargetOperatedLabelPrefix indicates a Target has finished operate phase - TargetOperatedLabelPrefix = "operated.opslifecycle.kusionstack.io" - // TargetPostCheckLabelPrefix indicates a Target is in post-check phase - TargetPostCheckLabelPrefix = "post-check.opslifecycle.kusionstack.io" - // TargetPostCheckedLabelPrefix indicates a Target has finished post-check phase - TargetPostCheckedLabelPrefix = "post-checked.opslifecycle.kusionstack.io" - // TargetCompletingLabelPrefix indicates a Target is completing operation - TargetCompletingLabelPrefix = "completing.opslifecycle.kusionstack.io" - - // TargetServiceAvailableLabel indicates a Target is available to serve - TargetServiceAvailableLabel = "opslifecycle.kusionstack.io/service-available" - TargetPreCheckLabel = "opslifecycle.kusionstack.io/pre-checking" - TargetPreparingLabel = "opslifecycle.kusionstack.io/preparing" - TargetOperatingLabel = "opslifecycle.kusionstack.io/operating" - TargetPostCheckLabel = "opslifecycle.kusionstack.io/post-checking" - TargetCompletingLabel = "opslifecycle.kusionstack.io/completing" - TargetCreatingLabel = "opslifecycle.kusionstack.io/creating" - - // TargetStayOfflineLabel indicates a Target is not ready and available to serve - TargetStayOfflineLabel = "opslifecycle.kusionstack.io/stay-offline" - TargetPreparingDeleteLabel = "opslifecycle.kusionstack.io/preparing-to-delete" -) - -var WellKnownLabelPrefixesWithID = []string{ - TargetOperatingLabelPrefix, - TargetOperationTypeLabelPrefix, - TargetPreCheckLabelPrefix, - TargetPreCheckedLabelPrefix, - TargetPreparingLabelPrefix, - TargetDoneOperationTypeLabelPrefix, - TargetUndoOperationTypeLabelPrefix, - TargetOperateLabelPrefix, - TargetOperatedLabelPrefix, - TargetPostCheckLabelPrefix, - TargetPostCheckedLabelPrefix, - TargetCompletingLabelPrefix, -} diff --git a/xset/opslifecycle/default_opslifecycle.go b/xset/opslifecycle/default_adapters.go similarity index 87% rename from xset/opslifecycle/default_opslifecycle.go rename to xset/opslifecycle/default_adapters.go index 804defa3..323b6466 100644 --- a/xset/opslifecycle/default_opslifecycle.go +++ b/xset/opslifecycle/default_adapters.go @@ -24,7 +24,9 @@ import ( var _ api.LifecycleAdapter = &DefaultUpdateLifecycleAdapter{} -type DefaultUpdateLifecycleAdapter struct{} +type DefaultUpdateLifecycleAdapter struct { + LabelManager api.LifeCycleLabelManager +} func (d *DefaultUpdateLifecycleAdapter) GetID() string { return "xset" @@ -39,7 +41,7 @@ func (d *DefaultUpdateLifecycleAdapter) AllowMultiType() bool { } func (d *DefaultUpdateLifecycleAdapter) WhenBegin(target client.Object) (bool, error) { - setOperate(d, target) + setOperate(d.LabelManager, d, target) return true, nil } @@ -49,7 +51,9 @@ func (d *DefaultUpdateLifecycleAdapter) WhenFinish(target client.Object) (bool, var _ api.LifecycleAdapter = &DefaultScaleInLifecycleAdapter{} -type DefaultScaleInLifecycleAdapter struct{} +type DefaultScaleInLifecycleAdapter struct { + LabelManager api.LifeCycleLabelManager +} func (d *DefaultScaleInLifecycleAdapter) GetID() string { return "xset" @@ -64,7 +68,7 @@ func (d *DefaultScaleInLifecycleAdapter) AllowMultiType() bool { } func (d *DefaultScaleInLifecycleAdapter) WhenBegin(target client.Object) (bool, error) { - setOperate(d, target) + setOperate(d.LabelManager, d, target) return true, nil } diff --git a/xset/opslifecycle/label_manager.go b/xset/opslifecycle/label_manager.go new file mode 100644 index 00000000..f3f4ef47 --- /dev/null +++ b/xset/opslifecycle/label_manager.go @@ -0,0 +1,72 @@ +/* + * Copyright 2024 - 2025 KusionStack Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package opslifecycle + +import ( + "kusionstack.io/kube-utils/xset/api" +) + +// OpsLifecycle default labels +var ( + defaultOperatingLabelPrefix = "operating.opslifecycle.kusionstack.io" + defaultOperationTypeLabelPrefix = "operation-type.opslifecycle.kusionstack.io" + defaultOperateLabelPrefix = "operate.opslifecycle.kusionstack.io" + defaultUndoOperationTypeLabelPrefix = "undo-operation-type.opslifecycle.kusionstack.io" + defaultServiceAvailableLabel = "opslifecycle.kusionstack.io/service-available" + defaultPreparingDeleteLabel = "opslifecycle.kusionstack.io/preparing-to-delete" +) + +var defaultLables = map[api.OperationLabelEnum]string{ + api.OperatingLabelPrefix: defaultOperatingLabelPrefix, + api.OperationTypeLabelPrefix: defaultOperationTypeLabelPrefix, + api.OperateLabelPrefix: defaultOperateLabelPrefix, + api.UndoOperationTypeLabelPrefix: defaultUndoOperationTypeLabelPrefix, + api.ServiceAvailableLabel: defaultServiceAvailableLabel, + api.PreparingDeleteLabel: defaultPreparingDeleteLabel, +} + +type LabelManagerImpl struct { + labels map[api.OperationLabelEnum]string + wellKnownLabelPrefixesWithID []string +} + +func NewLabelManager(overwrite map[api.OperationLabelEnum]string) api.LifeCycleLabelManager { + labelKeys := make(map[api.OperationLabelEnum]string) + for k, v := range defaultLables { + labelKeys[k] = v + } + if len(overwrite) > 0 { + for k, v := range overwrite { + labelKeys[k] = v + } + } + + wellKnownLabelPrefilxesWithID := []string{ + labelKeys[api.OperatingLabelPrefix], + labelKeys[api.OperationTypeLabelPrefix], + labelKeys[api.UndoOperationTypeLabelPrefix], + labelKeys[api.OperatingLabelPrefix], + } + return &LabelManagerImpl{ + labels: labelKeys, + wellKnownLabelPrefixesWithID: wellKnownLabelPrefilxesWithID, + } +} + +func (m *LabelManagerImpl) Get(labelType api.OperationLabelEnum) string { + return m.labels[labelType] +} diff --git a/xset/opslifecycle/utils.go b/xset/opslifecycle/utils.go index cfbe475f..4f56a03a 100644 --- a/xset/opslifecycle/utils.go +++ b/xset/opslifecycle/utils.go @@ -29,17 +29,23 @@ import ( "kusionstack.io/kube-utils/xset/api" ) +type UpdateFunc func(object client.Object) (bool, error) + +func IsServiceAvailable(m api.LifeCycleLabelManager, target client.Object) bool { + _, exists := target.GetLabels()[m.Get(api.ServiceAvailableLabel)] + return exists +} + // IDToLabelsMap returns a map of pod id to labels map and a map of operation type to number of pods. -func IDToLabelsMap(target client.Object) (map[string]map[string]string, map[string]int, error) { +func IDToLabelsMap(m *LabelManagerImpl, target client.Object) (map[string]map[string]string, map[string]int, error) { idToLabelsMap := map[string]map[string]string{} typeToNumsMap := map[string]int{} ids := sets.String{} labels := target.GetLabels() for k := range labels { - if strings.HasPrefix(k, TargetOperatingLabelPrefix) || - strings.HasPrefix(k, TargetOperateLabelPrefix) || - strings.HasPrefix(k, TargetOperatedLabelPrefix) { + if strings.HasPrefix(k, m.Get(api.OperatingLabelPrefix)) || + strings.HasPrefix(k, m.Get(api.OperateLabelPrefix)) { s := strings.Split(k, "/") if len(s) < 2 { return nil, nil, fmt.Errorf("invalid label %s", k) @@ -49,21 +55,11 @@ func IDToLabelsMap(target client.Object) (map[string]map[string]string, map[stri } for id := range ids { - for _, val := range []string{TargetOperationTypeLabelPrefix, TargetDoneOperationTypeLabelPrefix} { - operationType, ok := labels[fmt.Sprintf("%s/%s", val, id)] - if !ok { - continue - } - - if _, ok := typeToNumsMap[operationType]; !ok { - typeToNumsMap[operationType] = 1 - } else { - typeToNumsMap[operationType] += 1 - } - break + if operationType, ok := labels[fmt.Sprintf("%s/%s", m.Get(api.OperationTypeLabelPrefix), id)]; ok { + typeToNumsMap[operationType] += 1 } - for _, prefix := range WellKnownLabelPrefixesWithID { + for _, prefix := range m.wellKnownLabelPrefixesWithID { label := fmt.Sprintf("%s/%s", prefix, id) value, ok := labels[label] if !ok { @@ -82,16 +78,16 @@ func IDToLabelsMap(target client.Object) (map[string]map[string]string, map[stri } // NumOfLifecycleOnTarget returns the nums of lifecycles on pod -func NumOfLifecycleOnTarget(target client.Object) (int, error) { +func NumOfLifecycleOnTarget(m *LabelManagerImpl, target client.Object) (int, error) { if target == nil { return 0, nil } - newIDToLabelsMap, _, err := IDToLabelsMap(target) + newIDToLabelsMap, _, err := IDToLabelsMap(m, target) return len(newIDToLabelsMap), err } -var WhenBeginDelete api.UpdateFunc = func(obj client.Object) (bool, error) { - return AddLabel(obj, TargetPreparingDeleteLabel, strconv.FormatInt(time.Now().UnixNano(), 10)), nil +func WhenBeginDelete(m api.LifeCycleLabelManager, obj client.Object) (bool, error) { + return AddLabel(obj, m.Get(api.PreparingDeleteLabel), strconv.FormatInt(time.Now().UnixNano(), 10)), nil } func AddLabel(po client.Object, k, v string) bool { @@ -109,21 +105,21 @@ func AddLabel(po client.Object, k, v string) bool { // IsDuringOps decides whether the Target is during ops or not // DuringOps means the Target's OpsLifecycle phase is in or after PreCheck phase and before Finish phase. -func IsDuringOps(adapter api.LifecycleAdapter, obj client.Object) bool { - _, hasID := checkOperatingID(adapter, obj) - _, hasType := checkOperationType(adapter, obj) +func IsDuringOps(m api.LifeCycleLabelManager, adapter api.LifecycleAdapter, obj client.Object) bool { + _, hasID := checkOperatingID(m, adapter, obj) + _, hasType := checkOperationType(m, adapter, obj) return hasID && hasType } // Begin is used for an CRD Operator to begin a lifecycle -func Begin(c client.Client, adapter api.LifecycleAdapter, obj client.Object, updateFunc ...api.UpdateFunc) (updated bool, err error) { +func Begin(m api.LifeCycleLabelManager, c client.Client, adapter api.LifecycleAdapter, obj client.Object, updateFuncs ...UpdateFunc) (updated bool, err error) { if obj.GetLabels() == nil { obj.SetLabels(map[string]string{}) } - operatingID, hasID := checkOperatingID(adapter, obj) - operationType, hasType := checkOperationType(adapter, obj) + operatingID, hasID := checkOperatingID(m, adapter, obj) + operationType, hasType := checkOperationType(m, adapter, obj) var needUpdate bool // ensure operatingID and operationType @@ -134,7 +130,7 @@ func Begin(c client.Client, adapter api.LifecycleAdapter, obj client.Object, upd } } else { // check another id/type = this.type - currentTypeIDs := queryByOperationType(adapter, obj) + currentTypeIDs := queryByOperationType(m, adapter, obj) if currentTypeIDs != nil && currentTypeIDs.Len() > 0 && !adapter.AllowMultiType() { err = fmt.Errorf("operationType %s exists: %v", adapter.GetType(), currentTypeIDs) return updated, err @@ -142,15 +138,15 @@ func Begin(c client.Client, adapter api.LifecycleAdapter, obj client.Object, upd if !hasID { needUpdate = true - setOperatingID(adapter, obj) + setOperatingID(m, adapter, obj) } if !hasType { needUpdate = true - setOperationType(adapter, obj) + setOperationType(m, adapter, obj) } } - updated, err = DefaultUpdateAll(obj, append(updateFunc, adapter.WhenBegin)...) + updated, err = DefaultUpdateAll(obj, append(updateFuncs, adapter.WhenBegin)...) if err != nil { return updated, err } @@ -164,24 +160,24 @@ func Begin(c client.Client, adapter api.LifecycleAdapter, obj client.Object, upd } // BeginWithCleaningOld is used for an CRD Operator to begin a lifecycle with cleaning the old lifecycle -func BeginWithCleaningOld(c client.Client, adapter api.LifecycleAdapter, obj client.Object, updateFunc ...api.UpdateFunc) (updated bool, err error) { - if targetInUpdateLifecycle, err := IsLifecycleOnTarget(adapter.GetID(), obj); err != nil { +func BeginWithCleaningOld(m api.LifeCycleLabelManager, c client.Client, adapter api.LifecycleAdapter, obj client.Object, updateFunc ...UpdateFunc) (updated bool, err error) { + if targetInUpdateLifecycle, err := IsLifecycleOnTarget(m, adapter.GetID(), obj); err != nil { return false, fmt.Errorf("fail to check %s TargetOpsLifecycle on Target %s/%s: %w", adapter.GetID(), obj.GetNamespace(), obj.GetName(), err) } else if targetInUpdateLifecycle { - if err := Undo(c, adapter, obj); err != nil { + if err := Undo(m, c, adapter, obj); err != nil { return false, err } } - return Begin(c, adapter, obj, updateFunc...) + return Begin(m, c, adapter, obj, updateFunc...) } // AllowOps is used to check whether the TargetOpsLifecycle phase is in UPGRADE to do following operations. -func AllowOps(adapter api.LifecycleAdapter, operationDelaySeconds int32, obj client.Object) (requeueAfter *time.Duration, allow bool) { - if !IsDuringOps(adapter, obj) { +func AllowOps(m api.LifeCycleLabelManager, adapter api.LifecycleAdapter, operationDelaySeconds int32, obj client.Object) (requeueAfter *time.Duration, allow bool) { + if !IsDuringOps(m, adapter, obj) { return nil, false } - startedTimestampStr, started := checkOperate(adapter, obj) + startedTimestampStr, started := checkOperate(m, adapter, obj) if !started || operationDelaySeconds <= 0 { return nil, started } @@ -203,9 +199,9 @@ func AllowOps(adapter api.LifecycleAdapter, operationDelaySeconds int32, obj cli } // Finish is used for an CRD Operator to finish a lifecycle -func Finish(c client.Client, adapter api.LifecycleAdapter, obj client.Object, updateFuncs ...api.UpdateFunc) (updated bool, err error) { - operatingID, hasID := checkOperatingID(adapter, obj) - operationType, hasType := checkOperationType(adapter, obj) +func Finish(m api.LifeCycleLabelManager, c client.Client, adapter api.LifecycleAdapter, obj client.Object, updateFuncs ...UpdateFunc) (updated bool, err error) { + operatingID, hasID := checkOperatingID(m, adapter, obj) + operationType, hasType := checkOperationType(m, adapter, obj) if hasType && operationType != adapter.GetType() { return false, fmt.Errorf("operatingID %s has invalid operationType %s", operatingID, operationType) @@ -214,7 +210,7 @@ func Finish(c client.Client, adapter api.LifecycleAdapter, obj client.Object, up var needUpdate bool if hasID || hasType { needUpdate = true - deleteOperatingID(adapter, obj) + deleteOperatingID(m, adapter, obj) } updated, err = DefaultUpdateAll(obj, append(updateFuncs, adapter.WhenFinish)...) @@ -230,67 +226,66 @@ func Finish(c client.Client, adapter api.LifecycleAdapter, obj client.Object, up } // Undo is used for an CRD Operator to undo a lifecycle -func Undo(c client.Client, adapter api.LifecycleAdapter, obj client.Object) error { - setUndo(adapter, obj) +func Undo(m api.LifeCycleLabelManager, c client.Client, adapter api.LifecycleAdapter, obj client.Object) error { + setUndo(m, adapter, obj) return c.Update(context.Background(), obj) } -func checkOperatingID(adapter api.LifecycleAdapter, obj client.Object) (val string, ok bool) { - labelID := fmt.Sprintf("%s/%s", TargetOperatingLabelPrefix, adapter.GetID()) +func checkOperatingID(m api.LifeCycleLabelManager, adapter api.LifecycleAdapter, obj client.Object) (val string, ok bool) { + labelID := fmt.Sprintf("%s/%s", m.Get(api.OperatingLabelPrefix), adapter.GetID()) _, ok = obj.GetLabels()[labelID] return adapter.GetID(), ok } -func checkOperationType(adapter api.LifecycleAdapter, obj client.Object) (val api.OperationType, ok bool) { - labelType := fmt.Sprintf("%s/%s", TargetOperationTypeLabelPrefix, adapter.GetID()) +func checkOperationType(m api.LifeCycleLabelManager, adapter api.LifecycleAdapter, obj client.Object) (val api.OperationType, ok bool) { + labelType := fmt.Sprintf("%s/%s", m.Get(api.OperationTypeLabelPrefix), adapter.GetID()) labelVal := obj.GetLabels()[labelType] val = api.OperationType(labelVal) return val, val == adapter.GetType() } -func checkOperate(adapter api.LifecycleAdapter, obj client.Object) (val string, ok bool) { - labelOperate := fmt.Sprintf("%s/%s", TargetOperateLabelPrefix, adapter.GetID()) +func checkOperate(m api.LifeCycleLabelManager, adapter api.LifecycleAdapter, obj client.Object) (val string, ok bool) { + labelOperate := fmt.Sprintf("%s/%s", m.Get(api.OperateLabelPrefix), adapter.GetID()) val, ok = obj.GetLabels()[labelOperate] return } -func setOperatingID(adapter api.LifecycleAdapter, obj client.Object) { - labelID := fmt.Sprintf("%s/%s", TargetOperatingLabelPrefix, adapter.GetID()) +func setOperatingID(m api.LifeCycleLabelManager, adapter api.LifecycleAdapter, obj client.Object) { + labelID := fmt.Sprintf("%s/%s", m.Get(api.OperatingLabelPrefix), adapter.GetID()) obj.GetLabels()[labelID] = fmt.Sprintf("%d", time.Now().UnixNano()) return } -func setOperationType(adapter api.LifecycleAdapter, obj client.Object) { - labelType := fmt.Sprintf("%s/%s", TargetOperationTypeLabelPrefix, adapter.GetID()) +func setOperationType(m api.LifeCycleLabelManager, adapter api.LifecycleAdapter, obj client.Object) { + labelType := fmt.Sprintf("%s/%s", m.Get(api.OperationTypeLabelPrefix), adapter.GetID()) obj.GetLabels()[labelType] = string(adapter.GetType()) return } -// setOperate only for test -func setOperate(adapter api.LifecycleAdapter, obj client.Object) { - labelOperate := fmt.Sprintf("%s/%s", TargetOperateLabelPrefix, adapter.GetID()) - now := time.Now().UnixNano() - obj.GetLabels()[labelOperate] = strconv.FormatInt(now, 10) +// setOperate only for test, expected to be called by adapter +func setOperate(m api.LifeCycleLabelManager, adapter api.LifecycleAdapter, obj client.Object) { + labelOperate := fmt.Sprintf("%s/%s", m.Get(api.OperateLabelPrefix), adapter.GetID()) + obj.GetLabels()[labelOperate] = fmt.Sprintf("%d", time.Now().UnixNano()) return } -func setUndo(adapter api.LifecycleAdapter, obj client.Object) { - labelUndo := fmt.Sprintf("%s/%s", TargetUndoOperationTypeLabelPrefix, adapter.GetID()) +func setUndo(m api.LifeCycleLabelManager, adapter api.LifecycleAdapter, obj client.Object) { + labelUndo := fmt.Sprintf("%s/%s", m.Get(api.UndoOperationTypeLabelPrefix), adapter.GetID()) obj.GetLabels()[labelUndo] = string(adapter.GetType()) } -func deleteOperatingID(adapter api.LifecycleAdapter, obj client.Object) { - labelID := fmt.Sprintf("%s/%s", TargetOperatingLabelPrefix, adapter.GetID()) +func deleteOperatingID(m api.LifeCycleLabelManager, adapter api.LifecycleAdapter, obj client.Object) { + labelID := fmt.Sprintf("%s/%s", m.Get(api.OperatingLabelPrefix), adapter.GetID()) delete(obj.GetLabels(), labelID) return } -func queryByOperationType(adapter api.LifecycleAdapter, obj client.Object) sets.String { +func queryByOperationType(m api.LifeCycleLabelManager, adapter api.LifecycleAdapter, obj client.Object) sets.String { res := sets.String{} valType := adapter.GetType() for k, v := range obj.GetLabels() { - if strings.HasPrefix(k, TargetOperationTypeLabelPrefix) && v == string(valType) { + if strings.HasPrefix(k, m.Get(api.OperationTypeLabelPrefix)) && v == string(valType) { res.Insert(k) } } @@ -298,20 +293,9 @@ func queryByOperationType(adapter api.LifecycleAdapter, obj client.Object) sets. return res } -func DefaultUpdateAll(target client.Object, updateFuncs ...api.UpdateFunc) (updated bool, err error) { - for _, updateFunc := range updateFuncs { - ok, updateErr := updateFunc(target) - if updateErr != nil { - return updated, updateErr - } - updated = updated || ok - } - return updated, nil -} - -func IsLifecycleOnTarget(operatingID string, target client.Object) (bool, error) { +func IsLifecycleOnTarget(m api.LifeCycleLabelManager, operatingID string, target client.Object) (bool, error) { if target == nil { - return false, fmt.Errorf("nil Target") + return false, fmt.Errorf("nil target") } labels := target.GetLabels() @@ -319,9 +303,20 @@ func IsLifecycleOnTarget(operatingID string, target client.Object) (bool, error) return false, nil } - if val, ok := labels[fmt.Sprintf("%s/%s", TargetOperatingLabelPrefix, operatingID)]; ok { + if val, ok := labels[fmt.Sprintf("%s/%s", m.Get(api.OperatingLabelPrefix), operatingID)]; ok { return val != "", nil } return false, nil } + +func DefaultUpdateAll(target client.Object, updateFuncs ...UpdateFunc) (updated bool, err error) { + for _, updateFunc := range updateFuncs { + ok, updateErr := updateFunc(target) + if updateErr != nil { + return updated, updateErr + } + updated = updated || ok + } + return updated, nil +} diff --git a/xset/opslifecycle/utils_test.go b/xset/opslifecycle/utils_test.go index 70fc3b8a..5616adcb 100644 --- a/xset/opslifecycle/utils_test.go +++ b/xset/opslifecycle/utils_test.go @@ -55,6 +55,7 @@ func TestLifecycle(t *testing.T) { a := &mockAdapter{id: "id-1", operationType: "type-1"} b := &mockAdapter{id: "id-2", operationType: "type-1"} + mgr := NewLabelManager(nil) inputs := []struct { hasOperating, hasConflictID bool @@ -73,7 +74,7 @@ func TestLifecycle(t *testing.T) { { hasConflictID: true, started: false, - err: fmt.Errorf("operationType %s exists: %v", a.GetType(), sets.NewString(fmt.Sprintf("%s/%s", TargetOperationTypeLabelPrefix, b.GetID()))), + err: fmt.Errorf("operationType %s exists: %v", a.GetType(), sets.NewString(fmt.Sprintf("%s/%s", mgr.Get(api.OperationTypeLabelPrefix), b.GetID()))), }, { hasConflictID: true, @@ -97,34 +98,34 @@ func TestLifecycle(t *testing.T) { g.Expect(c.Create(context.TODO(), target)).Should(gomega.Succeed()) if input.hasOperating { - setOperatingID(a, target) - setOperationType(a, target) + setOperatingID(mgr, a, target) + setOperationType(mgr, a, target) a.WhenBegin(target) } if input.hasConflictID { - setOperatingID(b, target) - setOperationType(b, target) + setOperatingID(mgr, b, target) + setOperationType(mgr, b, target) } - _, err := Begin(c, a, target) + _, err := Begin(mgr, c, a, target) g.Expect(reflect.DeepEqual(err, input.err)).Should(gomega.BeTrue()) if err != nil { continue } g.Expect(target.Labels[mockLabelKey]).Should(gomega.BeEquivalentTo(mockLabelValue)) - setOperate(a, target) - started, err := Begin(c, a, target) + setOperate(mgr, a, target) + started, err := Begin(mgr, c, a, target) g.Expect(err).ShouldNot(gomega.HaveOccurred()) g.Expect(started).Should(gomega.BeTrue()) - g.Expect(IsDuringOps(a, target)).Should(gomega.BeTrue()) + g.Expect(IsDuringOps(mgr, a, target)).Should(gomega.BeTrue()) - finished, err := Finish(c, a, target) + finished, err := Finish(mgr, c, a, target) g.Expect(err).ShouldNot(gomega.HaveOccurred()) g.Expect(finished).Should(gomega.BeTrue()) g.Expect(target.Labels[mockLabelKey]).Should(gomega.BeEquivalentTo("")) - g.Expect(IsDuringOps(a, target)).Should(gomega.BeFalse()) + g.Expect(IsDuringOps(mgr, a, target)).Should(gomega.BeFalse()) } } diff --git a/xset/synccontrols/const.go b/xset/synccontrols/const.go index 0d70e3ea..a3fcf8b9 100644 --- a/xset/synccontrols/const.go +++ b/xset/synccontrols/const.go @@ -41,6 +41,8 @@ var ( TargetOrphanedIndicateLabelKey = "xset.kusionstack.io/orphaned" ) +// XSet annotations var ( + // LastTargetStatusAnnotationKey is used to attach last target status on target LastTargetStatusAnnotationKey = "xset.kusionstack.io/last-target-status" ) diff --git a/xset/synccontrols/sync_control.go b/xset/synccontrols/sync_control.go index 4e2a04bd..492a04a9 100644 --- a/xset/synccontrols/sync_control.go +++ b/xset/synccontrols/sync_control.go @@ -20,7 +20,6 @@ import ( "context" "errors" "fmt" - "strconv" "strings" "sync/atomic" "time" @@ -62,13 +61,17 @@ func NewRealSyncControl(reconcileMixIn *mixin.ReconcilerMixin, xControl xcontrol.TargetControl, cacheExpectation *expectations.CacheExpectation, ) SyncControl { + lifeCycleLabelManager := xsetController.GetLifeCycleLabelManager() + if lifeCycleLabelManager == nil { + lifeCycleLabelManager = opslifecycle.NewLabelManager(nil) + } scaleInOpsLicecycleAdapter := xsetController.GetScaleInOpsLifecycleAdapter() if scaleInOpsLicecycleAdapter == nil { - scaleInOpsLicecycleAdapter = &opslifecycle.DefaultScaleInLifecycleAdapter{} + scaleInOpsLicecycleAdapter = &opslifecycle.DefaultScaleInLifecycleAdapter{LabelManager: lifeCycleLabelManager} } updateLifecycleAdapter := xsetController.GetUpdateOpsLifecycleAdapter() if updateLifecycleAdapter == nil { - updateLifecycleAdapter = &opslifecycle.DefaultUpdateLifecycleAdapter{} + updateLifecycleAdapter = &opslifecycle.DefaultUpdateLifecycleAdapter{LabelManager: lifeCycleLabelManager} } xMeta := xsetController.XMeta() @@ -82,6 +85,7 @@ func NewRealSyncControl(reconcileMixIn *mixin.ReconcilerMixin, targetControl: xControl, recorder: reconcileMixIn.Recorder, + opsLifecycleMgr: lifeCycleLabelManager, scaleInLifecycleAdapter: scaleInOpsLicecycleAdapter, updateLifecycleAdapter: updateLifecycleAdapter, cacheExpectation: cacheExpectation, @@ -197,8 +201,8 @@ func (r *RealSyncControl) SyncTargets(ctx context.Context, instance api.XSetObje ToDelete: toDelete, ToExclude: toExclude, - IsDuringScaleInOps: opslifecycle.IsDuringOps(r.scaleInLifecycleAdapter, target), - IsDuringUpdateOps: opslifecycle.IsDuringOps(r.updateLifecycleAdapter, target), + IsDuringScaleInOps: opslifecycle.IsDuringOps(r.updateConfig.opsLifecycleMgr, r.scaleInLifecycleAdapter, target), + IsDuringUpdateOps: opslifecycle.IsDuringOps(r.updateConfig.opsLifecycleMgr, r.updateLifecycleAdapter, target), }) if id >= 0 { @@ -437,17 +441,10 @@ func (r *RealSyncControl) Scale(ctx context.Context, xsetObject api.XSetObject, // scale out new Targets with updatedRevision // TODO use cache target, err := NewTargetFrom(r.xsetController, xsetObject, revision, availableIDContext.ID) - - labels := target.GetLabels() - if availableIDContext.Data[resourcecontexts.JustCreateContextDataKey] == "true" { - labels[opslifecycle.TargetCreatingLabel] = strconv.FormatInt(time.Now().UnixNano(), 10) - } else { - labels[opslifecycle.TargetCompletingLabel] = strconv.FormatInt(time.Now().UnixNano(), 10) - } - if err != nil { return fmt.Errorf("fail to new Target from revision %s: %w", revision.GetName(), err) } + newTarget := target.DeepCopyObject().(client.Object) logger.V(1).Info("try to create Target with revision of "+r.xsetGVK.Kind, "revision", revision.GetName()) if target, err = r.xControl.CreateTarget(ctx, newTarget); err != nil { @@ -495,7 +492,7 @@ func (r *RealSyncControl) Scale(ctx context.Context, xsetObject api.XSetObject, // trigger TargetOpsLifecycle with scaleIn OperationType logger.V(1).Info("try to begin TargetOpsLifecycle for scaling in Target in XSet", "wrapper", ObjectKeyString(object)) // todo switch to x opslifecycle - if updated, err := opslifecycle.Begin(r.Client, r.scaleInLifecycleAdapter, object); err != nil { + if updated, err := opslifecycle.Begin(r.updateConfig.opsLifecycleMgr, r.Client, r.scaleInLifecycleAdapter, object); err != nil { return fmt.Errorf("fail to begin TargetOpsLifecycle for Scaling in Target %s/%s: %w", object.GetNamespace(), object.GetName(), err) } else if updated { r.Recorder.Eventf(object, corev1.EventTypeNormal, "BeginScaleInLifecycle", "succeed to begin TargetOpsLifecycle for scaling in") @@ -518,7 +515,7 @@ func (r *RealSyncControl) Scale(ctx context.Context, xsetObject api.XSetObject, needUpdateContext := false for i, targetWrapper := range targetsToScaleIn { - requeueAfter, allowed := opslifecycle.AllowOps(r.scaleInLifecycleAdapter, RealValue(spec.ScaleStrategy.OperationDelaySeconds), targetWrapper.Object) + requeueAfter, allowed := opslifecycle.AllowOps(r.updateConfig.opsLifecycleMgr, r.scaleInLifecycleAdapter, RealValue(spec.ScaleStrategy.OperationDelaySeconds), targetWrapper.Object) if !allowed && targetWrapper.Object.GetDeletionTimestamp() == nil { r.Recorder.Eventf(targetWrapper.Object, corev1.EventTypeNormal, "TargetScaleInLifecycle", "Target is not allowed to scale in") continue @@ -589,7 +586,7 @@ func (r *RealSyncControl) Scale(ctx context.Context, xsetObject api.XSetObject, needUpdateTargetContext := false for _, targetWrapper := range syncContext.activeTargets { if contextDetail, exist := syncContext.OwnedIds[targetWrapper.ID]; exist && contextDetail.Contains(resourcecontexts.ScaleInContextDataKey, "true") && - !opslifecycle.IsDuringOps(r.scaleInLifecycleAdapter, targetWrapper) { + !opslifecycle.IsDuringOps(r.updateConfig.opsLifecycleMgr, r.scaleInLifecycleAdapter, targetWrapper) { needUpdateTargetContext = true contextDetail.Remove(resourcecontexts.ScaleInContextDataKey) } @@ -639,7 +636,7 @@ func (r *RealSyncControl) Update(ctx context.Context, xsetObject api.XSetObject, continue } - if opslifecycle.IsDuringOps(r.updateLifecycleAdapter, targetInfo) { + if opslifecycle.IsDuringOps(r.updateConfig.opsLifecycleMgr, r.updateLifecycleAdapter, targetInfo) { continue } @@ -759,8 +756,8 @@ func (r *RealSyncControl) CalculateStatus(ctx context.Context, instance api.XSet updatedReplicas++ } - if opslifecycle.IsDuringOps(r.scaleInLifecycleAdapter, targetWrapper) || - opslifecycle.IsDuringOps(r.updateLifecycleAdapter, targetWrapper) { + if opslifecycle.IsDuringOps(r.updateConfig.opsLifecycleMgr, r.scaleInLifecycleAdapter, targetWrapper) || + opslifecycle.IsDuringOps(r.updateConfig.opsLifecycleMgr, r.updateLifecycleAdapter, targetWrapper) { operatingReplicas++ } diff --git a/xset/synccontrols/x_replace.go b/xset/synccontrols/x_replace.go index a5a58b50..e2f0b93b 100644 --- a/xset/synccontrols/x_replace.go +++ b/xset/synccontrols/x_replace.go @@ -143,7 +143,6 @@ func (r *RealSyncControl) replaceOriginTargets( ownedIDs[newTargetContext.ID].Remove(resourcecontexts.JustCreateContextDataKey) } newTarget.GetLabels()[TargetReplacePairOriginName] = originTarget.GetName() - newTarget.GetLabels()[opslifecycle.TargetCreatingLabel] = strconv.FormatInt(time.Now().UnixNano(), 10) newTargetContext.Put(resourcecontexts.RevisionContextDataKey, replaceRevision.GetName()) if newCreatedTarget, err := r.xControl.CreateTarget(ctx, newTarget); err == nil { @@ -199,7 +198,7 @@ func (r *RealSyncControl) dealReplaceTargets(targets []client.Object) (needRepla } // origin target is about to scaleIn, skip replace - if opslifecycle.IsDuringOps(r.scaleInLifecycleAdapter, target) { + if opslifecycle.IsDuringOps(r.updateConfig.opsLifecycleMgr, r.scaleInLifecycleAdapter, target) { continue } @@ -232,12 +231,12 @@ func (r *RealSyncControl) dealReplaceTargets(targets []client.Object) (needRepla needCleanLabels = append(needCleanLabels, TargetReplacePairOriginName) } else if originTarget.GetLabels()[TargetReplaceIndicationLabelKey] == "" { // replace canceled, delete replace new target if new target is not service available - if _, exist := targetLabels[opslifecycle.TargetServiceAvailableLabel]; !exist { + if serviceAvailable := opslifecycle.IsServiceAvailable(r.updateConfig.opsLifecycleMgr, target); !serviceAvailable { needDeleteTargets = append(needDeleteTargets, target) } } else if !replaceByUpdate { // not replace update, delete origin target when new created target is service available - if _, serviceAvailable := targetLabels[opslifecycle.TargetServiceAvailableLabel]; serviceAvailable { + if serviceAvailable := opslifecycle.IsServiceAvailable(r.updateConfig.opsLifecycleMgr, target); serviceAvailable { needDeleteTargets = append(needDeleteTargets, originTarget) } } diff --git a/xset/synccontrols/x_scale.go b/xset/synccontrols/x_scale.go index 0b3a68eb..29f0110e 100644 --- a/xset/synccontrols/x_scale.go +++ b/xset/synccontrols/x_scale.go @@ -61,7 +61,7 @@ func (r *RealSyncControl) getTargetsToDelete(filteredTargets []*targetWrapper, r continue } // when scaleIn origin Target, newTarget should be deleted if not service available - if _, serviceAvailable := replacePairTarget.GetLabels()[opslifecycle.TargetServiceAvailableLabel]; !serviceAvailable { + if serviceAvailable := opslifecycle.IsServiceAvailable(r.updateConfig.opsLifecycleMgr, target); !serviceAvailable { needDeleteTargets = append(needDeleteTargets, replacePairTarget) } } diff --git a/xset/synccontrols/x_update.go b/xset/synccontrols/x_update.go index 05f96a78..6017a56f 100644 --- a/xset/synccontrols/x_update.go +++ b/xset/synccontrols/x_update.go @@ -91,7 +91,7 @@ func (r *RealSyncControl) attachTargetUpdateInfo(xsetObject api.XSetObject, sync spec := r.xsetController.GetXSetSpec(xsetObject) // decide whether the TargetOpsLifecycle is during ops or not updateInfo.IsDuringOps = target.IsDuringUpdateOps - updateInfo.RequeueForOperationDelay, updateInfo.IsAllowOps = opslifecycle.AllowOps(r.updateLifecycleAdapter, RealValue(spec.UpdateStrategy.OperationDelaySeconds), target) + updateInfo.RequeueForOperationDelay, updateInfo.IsAllowOps = opslifecycle.AllowOps(r.updateConfig.opsLifecycleMgr, r.updateLifecycleAdapter, RealValue(spec.UpdateStrategy.OperationDelaySeconds), target) targetUpdateInfoList[i] = updateInfo } @@ -281,6 +281,7 @@ type UpdateConfig struct { targetControl xcontrol.TargetControl recorder record.EventRecorder + opsLifecycleMgr api.LifeCycleLabelManager scaleInLifecycleAdapter api.LifecycleAdapter updateLifecycleAdapter api.LifecycleAdapter @@ -314,9 +315,9 @@ func (u *GenericTargetUpdater) BeginUpdateTarget(_ context.Context, syncContext targetInfo := <-targetCh u.recorder.Eventf(targetInfo.Object, corev1.EventTypeNormal, "TargetUpdateLifecycle", "try to begin TargetOpsLifecycle for updating Target of XSet") - if updated, err := opslifecycle.BeginWithCleaningOld(u.client, u.updateLifecycleAdapter, targetInfo.Object, func(obj client.Object) (bool, error) { + if updated, err := opslifecycle.BeginWithCleaningOld(u.opsLifecycleMgr, u.client, u.updateLifecycleAdapter, targetInfo.Object, func(obj client.Object) (bool, error) { if !targetInfo.OnlyMetadataChanged && !targetInfo.InPlaceUpdateSupport { - return opslifecycle.WhenBeginDelete(obj) + return opslifecycle.WhenBeginDelete(u.opsLifecycleMgr, obj) } return false, nil }); err != nil { @@ -403,7 +404,7 @@ func (u *GenericTargetUpdater) FilterAllowOpsTargets(_ context.Context, candidat } func (u *GenericTargetUpdater) FinishUpdateTarget(_ context.Context, targetInfo *targetUpdateInfo) error { - if updated, err := opslifecycle.Finish(u.client, u.updateLifecycleAdapter, targetInfo.Object); err != nil { + if updated, err := opslifecycle.Finish(u.opsLifecycleMgr, u.client, u.updateLifecycleAdapter, targetInfo.Object); err != nil { return fmt.Errorf("failed to finish TargetOpsLifecycle for updating Target %s/%s: %s", targetInfo.GetNamespace(), targetInfo.GetName(), err.Error()) } else if updated { // add an expectation for this target update, before next reconciling @@ -659,7 +660,7 @@ func (u *replaceUpdateTargetUpdater) GetTargetUpdateFinishStatus(_ context.Conte return } - return isTargetUpdatedServiceAvailable(replaceNewTargetInfo) + return u.isTargetUpdatedServiceAvailable(replaceNewTargetInfo) } func (u *replaceUpdateTargetUpdater) FinishUpdateTarget(ctx context.Context, targetInfo *targetUpdateInfo) error { @@ -675,7 +676,7 @@ func (u *replaceUpdateTargetUpdater) FinishUpdateTarget(ctx context.Context, tar return nil } -func isTargetUpdatedServiceAvailable(targetInfo *targetUpdateInfo) (finished bool, msg string, err error) { +func (u *GenericTargetUpdater) isTargetUpdatedServiceAvailable(targetInfo *targetUpdateInfo) (finished bool, msg string, err error) { if targetInfo.GetLabels() == nil { return false, "no labels on target", nil } @@ -683,7 +684,7 @@ func isTargetUpdatedServiceAvailable(targetInfo *targetUpdateInfo) (finished boo return false, "replace origin target", nil } - if _, serviceAvailable := targetInfo.GetLabels()[opslifecycle.TargetServiceAvailableLabel]; serviceAvailable { + if serviceAvailable := opslifecycle.IsServiceAvailable(u.opsLifecycleMgr, targetInfo.Object); serviceAvailable { return true, "", nil } From b9062e5f74ca63b742e118d5cf99c7880a76f3b0 Mon Sep 17 00:00:00 2001 From: hexin <574252631@qq.com> Date: Wed, 23 Jul 2025 11:04:10 +0800 Subject: [PATCH 5/5] refactor: fix typos, use kube-api labels (#80) * use lifecycle labels from kube-api * fix lisence --- controller/utils/slow_start.go | 1 - xset/opslifecycle/label_manager.go | 22 ++++++++++++---------- xset/synccontrols/sync_control.go | 10 +++++----- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/controller/utils/slow_start.go b/controller/utils/slow_start.go index 5d8697dc..1668e1c7 100644 --- a/controller/utils/slow_start.go +++ b/controller/utils/slow_start.go @@ -1,5 +1,4 @@ /* -Copyright 2016 The Kubernetes Authors. Copyright 2024-2025 The KusionStack Authors. Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/xset/opslifecycle/label_manager.go b/xset/opslifecycle/label_manager.go index f3f4ef47..0de7e3dc 100644 --- a/xset/opslifecycle/label_manager.go +++ b/xset/opslifecycle/label_manager.go @@ -17,20 +17,22 @@ package opslifecycle import ( + appsv1alpha1 "kusionstack.io/kube-api/apps/v1alpha1" + "kusionstack.io/kube-utils/xset/api" ) // OpsLifecycle default labels var ( - defaultOperatingLabelPrefix = "operating.opslifecycle.kusionstack.io" - defaultOperationTypeLabelPrefix = "operation-type.opslifecycle.kusionstack.io" - defaultOperateLabelPrefix = "operate.opslifecycle.kusionstack.io" - defaultUndoOperationTypeLabelPrefix = "undo-operation-type.opslifecycle.kusionstack.io" - defaultServiceAvailableLabel = "opslifecycle.kusionstack.io/service-available" - defaultPreparingDeleteLabel = "opslifecycle.kusionstack.io/preparing-to-delete" + defaultOperatingLabelPrefix = appsv1alpha1.PodOperatingLabelPrefix + defaultOperationTypeLabelPrefix = appsv1alpha1.PodOperationTypeLabelPrefix + defaultOperateLabelPrefix = appsv1alpha1.PodOperateLabelPrefix + defaultUndoOperationTypeLabelPrefix = appsv1alpha1.PodUndoOperationTypeLabelPrefix + defaultServiceAvailableLabel = appsv1alpha1.PodServiceAvailableLabel + defaultPreparingDeleteLabel = appsv1alpha1.PodPreparingDeleteLabel ) -var defaultLables = map[api.OperationLabelEnum]string{ +var defaultLabels = map[api.OperationLabelEnum]string{ api.OperatingLabelPrefix: defaultOperatingLabelPrefix, api.OperationTypeLabelPrefix: defaultOperationTypeLabelPrefix, api.OperateLabelPrefix: defaultOperateLabelPrefix, @@ -46,7 +48,7 @@ type LabelManagerImpl struct { func NewLabelManager(overwrite map[api.OperationLabelEnum]string) api.LifeCycleLabelManager { labelKeys := make(map[api.OperationLabelEnum]string) - for k, v := range defaultLables { + for k, v := range defaultLabels { labelKeys[k] = v } if len(overwrite) > 0 { @@ -55,7 +57,7 @@ func NewLabelManager(overwrite map[api.OperationLabelEnum]string) api.LifeCycleL } } - wellKnownLabelPrefilxesWithID := []string{ + wellKnownLabelPrefixesWithID := []string{ labelKeys[api.OperatingLabelPrefix], labelKeys[api.OperationTypeLabelPrefix], labelKeys[api.UndoOperationTypeLabelPrefix], @@ -63,7 +65,7 @@ func NewLabelManager(overwrite map[api.OperationLabelEnum]string) api.LifeCycleL } return &LabelManagerImpl{ labels: labelKeys, - wellKnownLabelPrefixesWithID: wellKnownLabelPrefilxesWithID, + wellKnownLabelPrefixesWithID: wellKnownLabelPrefixesWithID, } } diff --git a/xset/synccontrols/sync_control.go b/xset/synccontrols/sync_control.go index 492a04a9..95ca01d1 100644 --- a/xset/synccontrols/sync_control.go +++ b/xset/synccontrols/sync_control.go @@ -65,9 +65,9 @@ func NewRealSyncControl(reconcileMixIn *mixin.ReconcilerMixin, if lifeCycleLabelManager == nil { lifeCycleLabelManager = opslifecycle.NewLabelManager(nil) } - scaleInOpsLicecycleAdapter := xsetController.GetScaleInOpsLifecycleAdapter() - if scaleInOpsLicecycleAdapter == nil { - scaleInOpsLicecycleAdapter = &opslifecycle.DefaultScaleInLifecycleAdapter{LabelManager: lifeCycleLabelManager} + scaleInOpsLifecycleAdapter := xsetController.GetScaleInOpsLifecycleAdapter() + if scaleInOpsLifecycleAdapter == nil { + scaleInOpsLifecycleAdapter = &opslifecycle.DefaultScaleInLifecycleAdapter{LabelManager: lifeCycleLabelManager} } updateLifecycleAdapter := xsetController.GetUpdateOpsLifecycleAdapter() if updateLifecycleAdapter == nil { @@ -86,7 +86,7 @@ func NewRealSyncControl(reconcileMixIn *mixin.ReconcilerMixin, recorder: reconcileMixIn.Recorder, opsLifecycleMgr: lifeCycleLabelManager, - scaleInLifecycleAdapter: scaleInOpsLicecycleAdapter, + scaleInLifecycleAdapter: scaleInOpsLifecycleAdapter, updateLifecycleAdapter: updateLifecycleAdapter, cacheExpectation: cacheExpectation, targetGVK: targetGVK, @@ -101,7 +101,7 @@ func NewRealSyncControl(reconcileMixIn *mixin.ReconcilerMixin, xsetGVK: xsetGVK, targetGVK: targetGVK, - scaleInLifecycleAdapter: scaleInOpsLicecycleAdapter, + scaleInLifecycleAdapter: scaleInOpsLifecycleAdapter, updateLifecycleAdapter: updateLifecycleAdapter, } }