diff --git a/api/core/v1alpha2/dvcr-deployment-condition/condition.go b/api/core/v1alpha2/dvcr-deployment-condition/condition.go new file mode 100644 index 0000000000..e1daba45a4 --- /dev/null +++ b/api/core/v1alpha2/dvcr-deployment-condition/condition.go @@ -0,0 +1,43 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dvcr_deployment_condition + +import appsv1 "k8s.io/api/apps/v1" + +const ( + // MaintenanceType indicates whether the deployment/dvcr is in maintenance mode. + MaintenanceType appsv1.DeploymentConditionType = "Maintenance" + + // LastMaintenanceResultType Type = "LastMaintenanceResult" +) + +type ( + // MaintenanceReason represents the various reasons for the DVCRMaintenance condition type. + MaintenanceReason string +) + +func (s MaintenanceReason) String() string { + return string(s) +} + +const ( + // InProgress indicates that the maintenance is in progress: wait for provisioners, or deployment is modified to run garbage collection. (status true) + InProgress MaintenanceReason = "GarbageCollectionInProgress" + + // LastResult indicates that the maintenance is done and result is in the message. (status false) + LastResult MaintenanceReason = "LastResult" +) diff --git a/api/core/v1alpha2/events.go b/api/core/v1alpha2/events.go index b5b4cd1a17..033cd3268e 100644 --- a/api/core/v1alpha2/events.go +++ b/api/core/v1alpha2/events.go @@ -104,6 +104,10 @@ const ( // ReasonDataSourceQuotaExceeded is event reason that DataSource sync is failed because quota exceed. ReasonDataSourceQuotaExceeded = "DataSourceQuotaExceed" + // ReasonImageOperationPostponedDueToDVCRMaintenance is event reason that operation is postponed until the end of DVCR maintenance mode. + ReasonImageOperationPostponedDueToDVCRMaintenance = "ImageOperationPostponedDueToDVCRMaintenance" + ReasonImageOperationContinueAfterDVCRMaintenance = "ImageOperationContinueAfterDVCRMaintenance" + // ReasonDataSourceDiskProvisioningFailed is event reason that DataSource disk provisioning is failed. ReasonDataSourceDiskProvisioningFailed = "DataSourceImportDiskProvisioningFailed" diff --git a/images/dvcr-artifact/cmd/dvcr-cleaner/cmd/gc.go b/images/dvcr-artifact/cmd/dvcr-cleaner/cmd/gc.go index 7645da3a56..f397bf6d35 100644 --- a/images/dvcr-artifact/cmd/dvcr-cleaner/cmd/gc.go +++ b/images/dvcr-artifact/cmd/dvcr-cleaner/cmd/gc.go @@ -17,13 +17,26 @@ limitations under the License. package cmd import ( + "context" + "encoding/json" "errors" "fmt" "os" "os/exec" + "sort" + "strings" + "time" + "github.com/hashicorp/go-multierror" "github.com/manifoldco/promptui" "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/api/resource" + + "github.com/deckhouse/virtualization-controller/dvcr-importers/pkg/cleaner/kubernetes" + "github.com/deckhouse/virtualization-controller/dvcr-importers/pkg/cleaner/registry" + "github.com/deckhouse/virtualization-controller/dvcr-importers/pkg/cleaner/signal" + "github.com/deckhouse/virtualization-controller/dvcr-importers/pkg/cleaner/storage" + "github.com/deckhouse/virtualization/api/core/v1alpha2" ) var GcCmd = &cobra.Command{ @@ -51,8 +64,7 @@ var gcRunCmd = &cobra.Command{ return fmt.Errorf("cache data cannot be deleted: %w", err) } - execCmd := exec.Command("registry", "garbage-collect", "/etc/docker/registry/config.yml", "--delete-untagged") - stdout, err := execCmd.Output() + stdout, err := registry.ExecGarbageCollect(context.Background()) if err != nil { fmt.Println(err.Error()) return nil @@ -65,6 +77,41 @@ var gcRunCmd = &cobra.Command{ SilenceErrors: true, } +var ( + MaintenanceSecretName string + GCTimeout time.Duration + GCTimeoutDefault = time.Minute * 10 +) + +var autoCleanupCmd = &cobra.Command{ + Use: "auto-cleanup [--maintenance-secret-name secret] [--gc-timeout duration]", + Short: "`auto-cleanup` deletes all stale images that have no corresponding resource in the cluster and then runs garbage-collect to remove underlying blobs (Note: not to be run with kubectl exec until you 100% sure what are you doing)", + Args: cobra.OnlyValidArgs, + RunE: autoCleanupHandler, + SilenceUsage: true, + SilenceErrors: true, +} + +var checkCmd = &cobra.Command{ + Use: "check", + Short: "`check` reports stale images that have no corresponding resource in the cluster", + Args: cobra.OnlyValidArgs, + RunE: checkCleanupHandler, + SilenceUsage: true, + SilenceErrors: true, +} + +func init() { + GcCmd.AddCommand(gcRunCmd) + + // Add 'run' command. + GcCmd.AddCommand(autoCleanupCmd) + autoCleanupCmd.Flags().StringVar(&MaintenanceSecretName, "maintenance-secret-name", "", "update secret with result and annotation after the cleanup") + autoCleanupCmd.Flags().DurationVar(&GCTimeout, "gc-timeout", GCTimeoutDefault, "max time for running garbage collection command") + // Add 'check' command. + GcCmd.AddCommand(checkCmd) +} + func Confirm() (bool, error) { prompt := promptui.Prompt{ Label: "Confirm", @@ -82,6 +129,267 @@ func Confirm() (bool, error) { return true, nil } -func init() { - GcCmd.AddCommand(gcRunCmd) +func autoCleanupHandler(cmd *cobra.Command, args []string) error { + fsInfoBeforeCleanup, err := registry.StorageStats() + if err != nil { + return fmt.Errorf("get repositories filesystem info before cleanup: %w", err) + } + + var errs *multierror.Error + + cleanupErr := performAutoCleanup() + if cleanupErr != nil { + errs = multierror.Append(errs, cleanupErr) + } + + // Report disk usage. + fsInfoAfterCleanup, errFSInfo := registry.StorageStats() + if errFSInfo != nil { + errs = multierror.Append(errs, fmt.Errorf("get repositories filesystem info after cleanup: %w", errFSInfo)) + } + freedSpace := "" + availableSpace := "" + usedSpace := "" + totalSpace := "" + if errFSInfo == nil { + // Available space after cleanup should be greater than available space before cleanup. + // The difference is the freed space. Format it with GiB/MiB suffix. + freedSpaceRaw := fsInfoAfterCleanup.Available - fsInfoBeforeCleanup.Available + freedSpace = storage.HumanizeQuantity(freedSpaceRaw) + "B" + availableSpace = storage.HumanizeQuantity(fsInfoAfterCleanup.Available) + "B" + usedSpace = storage.HumanizeQuantity(fsInfoAfterCleanup.Total-fsInfoAfterCleanup.Available) + "B" + totalSpace = storage.HumanizeQuantity(fsInfoAfterCleanup.Total) + "B" + } + fmt.Printf("Freed space during cleanup: %s, available space now: %s\n", freedSpace, availableSpace) + fmt.Printf("%7s %7s %7s\n", "Total", "Used", "Avail") + fmt.Printf("%7s %7s %7s\n", totalSpace, usedSpace, availableSpace) + + // Terminate without waiting if no secret name was provided. + if MaintenanceSecretName == "" { + return errs.ErrorOrNil() + } + + // Update maintenance secret and wait for termination signal. + result := map[string]string{ + "result": "success", + "freedSpace": freedSpace, + "availableSpace": availableSpace, + } + if cleanupErr != nil { + result["result"] = "fail" + result["error"] = cleanupErr.Error() + } + + secretErr := annotateMaintenanceSecretOnCleanupDone(context.Background(), result) + if secretErr != nil { + errs = multierror.Append(errs, secretErr) + } + + // Return previous errors, so Pod will be restarted without waiting. + err = errs.ErrorOrNil() + if err != nil { + return err + } + + // Wait until termination. + fmt.Println("Wait for signal before terminate.") + signal.WaitForTermination() + return nil +} + +func performAutoCleanup() error { + absentImages, err := getAbsentImages() + if err != nil { + return err + } + + // Delete manifests for absent images. + if len(absentImages) == 0 { + fmt.Println("No images eligible for cleanup.") + return nil + } + + err = registry.RemoveImages(absentImages) + if err != nil { + return fmt.Errorf("remove manifests: %w", err) + } + + // Run 'registry garbage-collect' to remove blobs. + gcContext, _ := context.WithTimeoutCause(context.Background(), GCTimeout, fmt.Errorf("garbage collect command is terminated, it runs more than %s", GCTimeout.String())) + stdout, err := registry.ExecGarbageCollect(gcContext) + errMsg := "" + if cause := context.Cause(gcContext); cause != nil { + errMsg = cause.Error() + "\n" + } + if err != nil { + if exitErr, ok := err.(*exec.ExitError); ok { + errMsg += fmt.Sprintf("Exit code: %d\nStderr: %s\n", exitErr.ExitCode(), exitErr.Stderr) + } else { + errMsg += err.Error() + } + } + + if errMsg != "" { + return errors.New(errMsg) + } + + fmt.Println(string(stdout)) + return nil +} + +func checkCleanupHandler(_ *cobra.Command, _ []string) error { + fsInfo, err := registry.StorageStats() + if err != nil { + return fmt.Errorf("get repositories filesystem info before cleanup: %w", err) + } + + absentImages, err := getAbsentImages() + if err != nil { + return err + } + + availableSpace := resource.NewQuantity(int64(fsInfo.Available), resource.BinarySI).String() + "B" + + fmt.Printf("Available space: %s\n", availableSpace) + + if len(absentImages) == 0 { + fmt.Println("No images eligible for auto-cleanup.") + } + + sort.SliceStable(absentImages, func(i, j int) bool { + return absentImages[i].Path < absentImages[j].Path + }) + + fmt.Println("Images eligible for cleanup:") + for _, image := range absentImages { + img := strings.TrimPrefix(image.Path, registry.RepoDir) + img = strings.TrimPrefix(image.Path, "/") + fmt.Println(img) + } + + return nil +} + +func getAbsentImages() ([]registry.Image, error) { + // List all images created for all ClusterVirtualImage and VirtualImage resources. + images, err := registry.ListImagesAll() + if err != nil { + return nil, fmt.Errorf("list all images: %w", err) + } + + // Get all VirtualDisks, VirtualImages, and ClusterVirtualImages + virtClient, err := kubernetes.NewVirtualizationClient() + if err != nil { + return nil, fmt.Errorf("initialize Kubernetes client: %w", err) + } + + kubeImages, err := virtClient.ListAllPossibleImages(context.Background()) + if err != nil { + return nil, fmt.Errorf("list images in cluster: %w", err) + } + + // Compare lists, return images absent in the cluster. + return compareRegistryAndClusterImages(images, kubeImages), nil +} + +// compareRegistryAndClusterImages returns images that has no corresponding resource in the cluster. +// VirtualDisks in Ready phase are considered for cleanup. +func compareRegistryAndClusterImages(images []registry.Image, kubeImages []kubernetes.ImageInfo) []registry.Image { + // Create indexes for all resources found in cluster. + // A map for ClusterImages. Keys are names. + clusterVirtualImages := make(map[string]struct{}) + // A map for virtualImages: namespace -> name + virtualImages := make(map[string]map[string]struct{}) + // A map for virtualDisks: namespace -> name -> disk phase + virtualDisks := make(map[string]map[string]v1alpha2.DiskPhase) + for _, kubeImage := range kubeImages { + switch kubeImage.Type { + case v1alpha2.ClusterVirtualImageKind: + clusterVirtualImages[kubeImage.Name] = struct{}{} + case v1alpha2.VirtualImageKind: + if _, ok := virtualImages[kubeImage.Namespace]; !ok { + virtualImages[kubeImage.Namespace] = make(map[string]struct{}) + } + virtualImages[kubeImage.Namespace][kubeImage.Name] = struct{}{} + case v1alpha2.VirtualDiskKind: + if _, ok := virtualDisks[kubeImage.Namespace]; !ok { + virtualDisks[kubeImage.Namespace] = make(map[string]v1alpha2.DiskPhase) + } + virtualDisks[kubeImage.Namespace][kubeImage.Name] = kubeImage.Phase + } + } + + absentImages := make([]registry.Image, 0) + for _, image := range images { + switch image.Type { + case v1alpha2.ClusterVirtualImageKind: + if _, ok := clusterVirtualImages[image.Name]; !ok { + absentImages = append(absentImages, image) + } + case v1alpha2.VirtualImageKind: + if _, ok := virtualImages[image.Namespace]; !ok { + absentImages = append(absentImages, image) + continue + } + if _, ok := virtualImages[image.Namespace][image.Name]; !ok { + absentImages = append(absentImages, image) + } + case v1alpha2.VirtualDiskKind: + if _, ok := virtualDisks[image.Namespace]; !ok { + absentImages = append(absentImages, image) + continue + } + if _, ok := virtualDisks[image.Namespace][image.Name]; !ok { + absentImages = append(absentImages, image) + continue + } + // Images for disks in Ready phase are eligible for cleanup. + if virtualDisks[image.Namespace][image.Name] == v1alpha2.DiskReady { + absentImages = append(absentImages, image) + } + } + } + + return absentImages +} + +const ( + garbageCollectionDoneAnno = "virtualization.deckhouse.io/dvcr-garbage-collection-done" + switchToMaintenanceAnno = "virtualization.deckhouse.io/dvcr-deployment-switch-to-maintenance-mode" +) + +func annotateMaintenanceSecretOnCleanupDone(ctx context.Context, result map[string]string) error { + resultBytes, err := json.Marshal(result) + if err != nil { + return fmt.Errorf("marshal result to json: %w", err) + } + + // Get all VirtualImages and ClusterImages + virtClient, err := kubernetes.NewVirtualizationClient() + if err != nil { + return fmt.Errorf("initialize Kubernetes client: %w", err) + } + + secret, err := virtClient.GetMaintenanceSecret(ctx) + if err != nil { + return err + } + + if secret.Annotations == nil { + secret.Annotations = make(map[string]string) + } + secret.Annotations[garbageCollectionDoneAnno] = "" + delete(secret.Annotations, switchToMaintenanceAnno) + + if secret.Data == nil { + secret.Data = make(map[string][]byte) + } + secret.Data["result"] = resultBytes + + err = virtClient.UpdateMaintenanceSecret(ctx, secret) + if err != nil { + return fmt.Errorf("update secret on cleanup done: %w", err) + } + + return nil } diff --git a/images/dvcr-artifact/cmd/dvcr-cleaner/cmd/ls.go b/images/dvcr-artifact/cmd/dvcr-cleaner/cmd/ls.go index 27fc844aaf..1e14658d0a 100644 --- a/images/dvcr-artifact/cmd/dvcr-cleaner/cmd/ls.go +++ b/images/dvcr-artifact/cmd/dvcr-cleaner/cmd/ls.go @@ -22,8 +22,9 @@ import ( "os" "text/tabwriter" - "github.com/deckhouse/virtualization/api/core/v1alpha2" "github.com/spf13/cobra" + + "github.com/deckhouse/virtualization/api/core/v1alpha2" ) var ( @@ -68,6 +69,7 @@ var lsCviCmd = &cobra.Command{ cobra.OnlyValidArgs, ), RunE: func(cmd *cobra.Command, args []string) error { + // return ListImage(v1alpha2.ClusterVirtualImageKind, cmd, args) imgsDir := fmt.Sprintf("%s/cvi", RepoDir) err := ListImage(v1alpha2.ClusterVirtualImageKind, imgsDir, cmd, args) if err != nil { @@ -86,6 +88,7 @@ func ListImage(imgType, imgsDir string, cmd *cobra.Command, args []string) error } if len(args) != 0 { + var ( fileInfo os.FileInfo err error @@ -113,6 +116,7 @@ func ListImage(imgType, imgsDir string, cmd *cobra.Command, args []string) error default: return fmt.Errorf("unknown image type: %s", imgType) } + w := tabwriter.NewWriter(os.Stdout, 1, 1, 1, ' ', 0) fmt.Fprintln(w, "Name\t") fmt.Fprintf(w, "%s\t\n", fileInfo.Name()) diff --git a/images/dvcr-artifact/go.mod b/images/dvcr-artifact/go.mod index ce1b1dfb34..bb20529b36 100644 --- a/images/dvcr-artifact/go.mod +++ b/images/dvcr-artifact/go.mod @@ -17,7 +17,10 @@ require ( github.com/prometheus/client_golang v1.19.0 github.com/prometheus/client_model v0.6.0 github.com/spf13/cobra v1.8.1 + github.com/spf13/pflag v1.0.5 golang.org/x/sync v0.14.0 + golang.org/x/sys v0.33.0 + k8s.io/apimachinery v0.30.2 k8s.io/klog/v2 v2.120.1 kubevirt.io/containerized-data-importer v0.0.0-00010101000000-000000000000 kubevirt.io/containerized-data-importer-api v1.60.3 @@ -64,9 +67,11 @@ require ( github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/gorilla/mux v1.8.1 // indirect + github.com/gorilla/websocket v1.5.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/imdario/mergo v0.3.16 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -95,7 +100,6 @@ require ( github.com/prometheus/common v0.51.1 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/sirupsen/logrus v1.9.3 // indirect - github.com/spf13/pflag v1.0.5 // indirect github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect github.com/ulikunitz/xz v0.5.15 // indirect github.com/vbatts/tar-split v0.11.5 // indirect @@ -110,7 +114,6 @@ require ( golang.org/x/exp v0.0.0-20240613232115-7f521ea00fb8 // indirect golang.org/x/net v0.26.0 // indirect golang.org/x/oauth2 v0.27.0 // indirect - golang.org/x/sys v0.33.0 // indirect golang.org/x/term v0.32.0 // indirect golang.org/x/text v0.25.0 // indirect golang.org/x/time v0.5.0 // indirect @@ -125,7 +128,6 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/api v0.30.2 // indirect k8s.io/apiextensions-apiserver v0.30.2 // indirect - k8s.io/apimachinery v0.30.2 // indirect k8s.io/apiserver v0.30.2 // indirect k8s.io/client-go v8.0.0+incompatible // indirect k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect diff --git a/images/dvcr-artifact/go.sum b/images/dvcr-artifact/go.sum index a7bb733a12..8c42c0333d 100644 --- a/images/dvcr-artifact/go.sum +++ b/images/dvcr-artifact/go.sum @@ -203,6 +203,8 @@ github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB7 github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= +github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= diff --git a/images/dvcr-artifact/pkg/cleaner/kubernetes/list.go b/images/dvcr-artifact/pkg/cleaner/kubernetes/list.go new file mode 100644 index 0000000000..301a01b9c6 --- /dev/null +++ b/images/dvcr-artifact/pkg/cleaner/kubernetes/list.go @@ -0,0 +1,169 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubernetes + +import ( + "context" + "fmt" + + "github.com/spf13/pflag" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + + "github.com/deckhouse/virtualization/api/client/kubeclient" + "github.com/deckhouse/virtualization/api/core/v1alpha2" +) + +type Client struct { + virtClient kubeclient.Client + kubeClient *kubernetes.Clientset +} + +func NewVirtualizationClient() (*Client, error) { + clientConfig := kubeclient.DefaultClientConfig(&pflag.FlagSet{}) + client, err := kubeclient.GetClientFromClientConfig(clientConfig) + if err != nil { + return nil, fmt.Errorf("init client for virtualization API: %w", err) + } + + restConfig, err := clientConfig.ClientConfig() + if err != nil { + return nil, fmt.Errorf("init rest config for kubernetes API: %w", err) + } + kubeClient, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return nil, fmt.Errorf("init client for kubernetes API: %w", err) + } + + return &Client{ + kubeClient: kubeClient, + virtClient: client, + }, nil +} + +type ImageInfo struct { + Type string + Namespace string + Name string + Phase v1alpha2.DiskPhase +} + +func (c *Client) ListAllPossibleImages(ctx context.Context) ([]ImageInfo, error) { + clusterVirtualImages, err := c.ListClusterVirtualImages(ctx) + if err != nil { + return nil, err + } + + virtualImages, err := c.ListVirtualImagesAll(ctx) + if err != nil { + return nil, err + } + + virtualDisks, err := c.ListVirtualDisksAll(ctx) + if err != nil { + return nil, err + } + + fmt.Printf("Found %d cvi, %d vi, %d vd resources in cluster\n", + len(clusterVirtualImages), + len(virtualImages), + len(virtualDisks), + ) + // Return all 3 arrays. + clusterVirtualImages = append(clusterVirtualImages, virtualImages...) + return append(clusterVirtualImages, virtualDisks...), nil +} + +func (c *Client) ListClusterVirtualImages(ctx context.Context) ([]ImageInfo, error) { + resources, err := c.virtClient.ClusterVirtualImages().List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, err + } + + images := make([]ImageInfo, 0, len(resources.Items)) + for _, resource := range resources.Items { + image := ImageInfo{ + Type: v1alpha2.ClusterVirtualImageKind, + Name: resource.GetName(), + } + images = append(images, image) + } + return images, nil +} + +func (c *Client) ListVirtualImagesAll(ctx context.Context) ([]ImageInfo, error) { + resources, err := c.virtClient.VirtualImages("").List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, err + } + + images := make([]ImageInfo, 0, len(resources.Items)) + for _, resource := range resources.Items { + image := ImageInfo{ + Type: v1alpha2.VirtualImageKind, + Namespace: resource.GetNamespace(), + Name: resource.GetName(), + } + images = append(images, image) + } + return images, nil +} + +func (c *Client) ListVirtualDisksAll(ctx context.Context) ([]ImageInfo, error) { + resources, err := c.virtClient.VirtualDisks("").List(ctx, metav1.ListOptions{}) + if err != nil { + return nil, err + } + + images := make([]ImageInfo, 0, len(resources.Items)) + for _, resource := range resources.Items { + image := ImageInfo{ + Type: v1alpha2.VirtualDiskKind, + Namespace: resource.GetNamespace(), + Name: resource.GetName(), + Phase: resource.Status.Phase, + } + images = append(images, image) + } + return images, nil +} + +const ( + maintenanceSecretNS = "d8-virtualization" + maintenanceSecretName = "dvcr-maintenance" +) + +func (c *Client) GetMaintenanceSecret(ctx context.Context) (*corev1.Secret, error) { + secret, err := c.kubeClient.CoreV1().Secrets(maintenanceSecretNS).Get(ctx, maintenanceSecretName, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("get maintenance secret: %w", err) + } + + return secret, nil +} + +func (c *Client) UpdateMaintenanceSecret(ctx context.Context, secret *corev1.Secret) error { + _, err := c.kubeClient.CoreV1().Secrets(maintenanceSecretNS). + Update(ctx, secret, metav1.UpdateOptions{}) + + if err != nil { + return fmt.Errorf("annotate maintenance secret: %w", err) + } + + return nil +} diff --git a/images/dvcr-artifact/pkg/cleaner/kubernetes/termination_log.go b/images/dvcr-artifact/pkg/cleaner/kubernetes/termination_log.go new file mode 100644 index 0000000000..28a9586bb7 --- /dev/null +++ b/images/dvcr-artifact/pkg/cleaner/kubernetes/termination_log.go @@ -0,0 +1,62 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package kubernetes + +import ( + "encoding/json" + "errors" + "fmt" + "os" +) + +const ( + PodTerminationMessageFile = "/dev/termination-log" +) + +var ErrFailedTerminationMessage = errors.New("failed to write termination message") + +func ReportTerminationMessage(err error, extra ...map[string]string) error { + messageMap := map[string]string{} + if err != nil { + messageMap["error"] = err.Error() + } + for _, extraMap := range extra { + for k, v := range extraMap { + messageMap[k] = v + } + } + + message, err := json.Marshal(messageMap) + if err != nil { + return fmt.Errorf("%w: %w", ErrFailedTerminationMessage, err) + } + + err = writeTerminationMessage(message) + if err != nil { + return fmt.Errorf("%w: %w", ErrFailedTerminationMessage, err) + } + + return nil +} + +func writeTerminationMessage(message []byte) error { + err := os.WriteFile(PodTerminationMessageFile, message, 0600) + if err != nil { + return fmt.Errorf("write termination message to %s: %w", err) + } + return nil +} diff --git a/images/dvcr-artifact/pkg/cleaner/registry/gc.go b/images/dvcr-artifact/pkg/cleaner/registry/gc.go new file mode 100644 index 0000000000..00af61bdac --- /dev/null +++ b/images/dvcr-artifact/pkg/cleaner/registry/gc.go @@ -0,0 +1,27 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registry + +import ( + "context" + "os/exec" +) + +func ExecGarbageCollect(ctx context.Context) ([]byte, error) { + execCmd := exec.CommandContext(ctx, "registry", "garbage-collect", "/etc/docker/registry/config.yml", "--delete-untagged") + return execCmd.Output() +} diff --git a/images/dvcr-artifact/pkg/cleaner/registry/list.go b/images/dvcr-artifact/pkg/cleaner/registry/list.go new file mode 100644 index 0000000000..e2f2691414 --- /dev/null +++ b/images/dvcr-artifact/pkg/cleaner/registry/list.go @@ -0,0 +1,264 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registry + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + + "github.com/deckhouse/virtualization/api/core/v1alpha2" +) + +const ( + RepoDir = "/var/lib/registry/docker/registry/v2/repositories" +) + +type Image struct { + Type string + Namespace string + Name string + Path string +} + +// clusterVirtualImagesDir returns a directory where stored all images for ClusterVirtualImage resources. +// +// Example: +// +// /.../repositories +// `-- cvi +// |-- ubuntu-24-04 +// `-- alpine-latest +func clusterVirtualImagesDir() string { + return filepath.Join(RepoDir, "cvi") +} + +// virtualImagesDir returns a directory where stored all images for VirtualImage resources. +// These images are actually one level deep, there is a directory with the corresponding namespace name. +// +// Example: +// +// /.../repositories +// `-- vi +// |-- default +// | `-- ubuntu-24-04 +// |-- testvms +// |-- alpine-latest +// |-- win-server +// `-- ubuntu-latest +func virtualImagesDir() string { + return filepath.Join(RepoDir, "vi") +} + +// virtualImagesDir returns a directory where stored all temporary images for VirtualDisk resources. +// Directory structure is the same as for VirtualImage (see virtualImagesDir). +func virtualDisksDir() string { + return filepath.Join(RepoDir, "vd") +} + +// ListImagesAll return image info for all image types: +// virtualImages, clusterVirtualImages and virtualDisks. +func ListImagesAll() ([]Image, error) { + clusterVirtualImages, err := ListClusterVirtualImages() + if err != nil { + return nil, err + } + + virtualImages, err := ListVirtualImagesAll() + if err != nil { + return nil, err + } + + virtualDiskImages, err := ListVirtualDisksAll() + if err != nil { + return nil, err + } + + fmt.Printf("Found %d cvi, %d vi, %d vd manifests in registry\n", + len(clusterVirtualImages), + len(virtualImages), + len(virtualDiskImages), + ) + + clusterVirtualImages = append(clusterVirtualImages, virtualImages...) + return append(clusterVirtualImages, virtualDiskImages...), nil +} + +func ListClusterVirtualImages() ([]Image, error) { + imagesDir := clusterVirtualImagesDir() + + imagesEntries, err := os.ReadDir(imagesDir) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + return nil, fmt.Errorf("cannot get the list of all `ClusterVirtualImages`: %w", err) + } + + images := make([]Image, 0) + for _, imageEntry := range imagesEntries { + images = append(images, Image{ + Type: v1alpha2.ClusterVirtualImageKind, + Namespace: "", + Name: imageEntry.Name(), + Path: filepath.Join(imagesDir, imageEntry.Name()), + }) + } + return images, nil +} + +func ListVirtualImagesAll() ([]Image, error) { + imagesDir := virtualImagesDir() + + namespacesEntries, err := os.ReadDir(imagesDir) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + return nil, fmt.Errorf("cannot list directories for namespaces in %s: %w", imagesDir, err) + } + + images := make([]Image, 0) + for _, nsEntry := range namespacesEntries { + nsImages, err := ListVirtualImagesForNamespace(nsEntry.Name()) + if err != nil { + return nil, err + } + images = append(images, nsImages...) + } + return images, nil +} + +func ListVirtualImagesForNamespace(namespace string) ([]Image, error) { + if namespace == "" { + namespace = "default" + } + + imagesDir := filepath.Join(virtualImagesDir(), namespace) + imagesEntries, err := os.ReadDir(imagesDir) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + return nil, fmt.Errorf("cannot list directories with images for `VirtualImage` resources in %s: %w", imagesDir, err) + } + + images := make([]Image, 0) + for _, imageEntry := range imagesEntries { + images = append(images, Image{ + Type: v1alpha2.VirtualImageKind, + Namespace: namespace, + Name: imageEntry.Name(), + Path: filepath.Join(imagesDir, imageEntry.Name()), + }) + } + return images, nil +} + +func GetAnyImage(imageType, namespace, name string) (*Image, error) { + var ( + imageDir string + fileInfo os.FileInfo + err error + ) + + switch imageType { + case v1alpha2.VirtualImageKind: + if namespace == "" { + namespace = "default" + } + imageDir = virtualImagesDir() + path := filepath.Join(imageDir, namespace, name) + fileInfo, err = os.Stat(path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("the `%s` %q is not found in %q namespace", imageType, name, namespace) + } + return nil, fmt.Errorf("cannot get the `%s` %q in the %q namespace: %w", imageType, name, namespace, err) + } + case v1alpha2.ClusterVirtualImageKind: + imageDir = clusterVirtualImagesDir() + path := filepath.Join(imageDir, name) + + fileInfo, err = os.Stat(path) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, fmt.Errorf("the `%s` %q is not found", imageType, name) + } + return nil, fmt.Errorf("cannot get the `%s` %q: %w", imageType, name, err) + } + default: + return nil, fmt.Errorf("unknown image type: %s", imageType) + } + + return &Image{ + Type: imageType, + Namespace: namespace, + Name: name, + Path: filepath.Join(imageDir, fileInfo.Name()), + }, nil +} + +func ListVirtualDisksAll() ([]Image, error) { + disksDir := virtualDisksDir() + + namespacesEntries, err := os.ReadDir(disksDir) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + return nil, fmt.Errorf("cannot list directories for namespaces in %s: %w", disksDir, err) + } + + disks := make([]Image, 0) + for _, nsEntry := range namespacesEntries { + disksInNamespace, err := ListVirtualDisksForNamespace(nsEntry.Name()) + if err != nil { + return nil, err + } + disks = append(disks, disksInNamespace...) + } + return disks, nil +} + +func ListVirtualDisksForNamespace(namespace string) ([]Image, error) { + if namespace == "" { + namespace = "default" + } + + disksDir := filepath.Join(virtualDisksDir(), namespace) + disksEntries, err := os.ReadDir(disksDir) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return nil, nil + } + return nil, fmt.Errorf("cannot list directories with images for `VirtualDisk` resources in %s: %w", disksDir, err) + } + + disksImages := make([]Image, 0) + for _, diskEntry := range disksEntries { + disksImages = append(disksImages, Image{ + Type: v1alpha2.VirtualDiskKind, + Namespace: namespace, + Name: diskEntry.Name(), + Path: filepath.Join(disksDir, diskEntry.Name()), + }) + } + return disksImages, nil +} diff --git a/images/dvcr-artifact/pkg/cleaner/registry/remove.go b/images/dvcr-artifact/pkg/cleaner/registry/remove.go new file mode 100644 index 0000000000..dac89401d2 --- /dev/null +++ b/images/dvcr-artifact/pkg/cleaner/registry/remove.go @@ -0,0 +1,62 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registry + +import ( + "fmt" + "os" + + "github.com/deckhouse/virtualization/api/core/v1alpha2" +) + +func RemoveImages(images []Image) error { + if len(images) == 0 { + return nil + } + + for _, image := range images { + err := RemoveImage(image) + if err != nil { + return err + } + } + + return nil +} + +func RemoveImage(image Image) error { + fmt.Printf("Remove manifest in %s directory\n", image.Path) + if _, err := os.Stat(image.Path); err != nil { + if os.IsNotExist(err) { + return fmt.Errorf("image directory %s for `%s` %q is not found", image.Path, image.Type, image.Name) + } + } + + err := os.RemoveAll(image.Path) + if err != nil { + switch image.Type { + case v1alpha2.VirtualImageKind, v1alpha2.VirtualDiskKind: + return fmt.Errorf("delete image directory %s for `%s` %q in %q namespace: %w", image.Path, image.Type, image.Name, image.Namespace, err) + case v1alpha2.ClusterVirtualImageKind: + return fmt.Errorf("delete image directory %s for `%s` %q: %w", image.Path, image.Type, image.Name, err) + default: + return fmt.Errorf("unknown image type: %s", image.Type) + } + } + + return nil +} diff --git a/images/dvcr-artifact/pkg/cleaner/registry/storage_stats.go b/images/dvcr-artifact/pkg/cleaner/registry/storage_stats.go new file mode 100644 index 0000000000..bb222e2b2a --- /dev/null +++ b/images/dvcr-artifact/pkg/cleaner/registry/storage_stats.go @@ -0,0 +1,23 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package registry + +import "github.com/deckhouse/virtualization-controller/dvcr-importers/pkg/cleaner/storage" + +func StorageStats() (storage.FSInfo, error) { + return storage.FSBytes(RepoDir) +} diff --git a/images/dvcr-artifact/pkg/cleaner/signal/wait.go b/images/dvcr-artifact/pkg/cleaner/signal/wait.go new file mode 100644 index 0000000000..6ec40d9d5e --- /dev/null +++ b/images/dvcr-artifact/pkg/cleaner/signal/wait.go @@ -0,0 +1,41 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package signal + +import ( + "fmt" + "os" + "os/signal" + "syscall" +) + +func WaitForTermination() { + exitCh := make(chan os.Signal, 5) + signal.Notify(exitCh, syscall.SIGINT, syscall.SIGTERM) + + for { + select { + case sig := <-exitCh: + fmt.Printf("Signal %q received, terminate now.", sig.String()) + signum := 0 + if v, ok := sig.(syscall.Signal); ok { + signum = int(v) + } + os.Exit(128 + signum) + } + } +} diff --git a/images/dvcr-artifact/pkg/cleaner/storage/stats.go b/images/dvcr-artifact/pkg/cleaner/storage/stats.go new file mode 100644 index 0000000000..629bc54283 --- /dev/null +++ b/images/dvcr-artifact/pkg/cleaner/storage/stats.go @@ -0,0 +1,56 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package storage + +import ( + "fmt" + "math" + "strconv" +) + +type FSInfo struct { + Total uint64 + Available uint64 +} + +func HumanizeQuantity(q uint64) string { + suffixes := []string{"B", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei"} + return humanizeQuantity(q, 1024, suffixes) +} + +func humanizeQuantity(s uint64, base float64, suffixes []string) string { + if s < 1200 { + return strconv.FormatUint(s, 10) + } + + e := math.Floor(logn(float64(s), base)) + + suffix := suffixes[int(e)] + val := math.Floor(float64(s)/math.Pow(base, e)*10+0.5) / 10 + _, frac := math.Modf(val) + f := "%.0f%s" + + if frac > 0 { + f = "%.1f%s" + } + + return fmt.Sprintf(f, val, suffix) +} + +func logn(n, b float64) float64 { + return math.Log(n) / math.Log(b) +} diff --git a/images/dvcr-artifact/pkg/cleaner/storage/stats_linux.go b/images/dvcr-artifact/pkg/cleaner/storage/stats_linux.go new file mode 100644 index 0000000000..c9e18a2f9f --- /dev/null +++ b/images/dvcr-artifact/pkg/cleaner/storage/stats_linux.go @@ -0,0 +1,38 @@ +//go:build linux + +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package storage + +import ( + "golang.org/x/sys/unix" +) + +func FSBytes(dir string) (FSInfo, error) { + var stat unix.Statfs_t + err := unix.Statfs(dir, &stat) + if err != nil { + return FSInfo{}, err + } + + return FSInfo{ + // + Total: stat.Blocks * uint64(stat.Bsize), + // Available blocks * size per block = available space in bytes. + Available: stat.Bavail * uint64(stat.Bsize), + }, nil +} diff --git a/images/dvcr-artifact/pkg/cleaner/storage/stats_other.go b/images/dvcr-artifact/pkg/cleaner/storage/stats_other.go new file mode 100644 index 0000000000..c034ee06a7 --- /dev/null +++ b/images/dvcr-artifact/pkg/cleaner/storage/stats_other.go @@ -0,0 +1,23 @@ +//go:build !linux + +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package storage + +func FSBytes(dir string) (FSInfo, error) { + return FSInfo{}, nil +} diff --git a/images/hooks/cmd/virtualization-module-hooks/register.go b/images/hooks/cmd/virtualization-module-hooks/register.go index 39c8b95a04..8924b2c1ef 100644 --- a/images/hooks/cmd/virtualization-module-hooks/register.go +++ b/images/hooks/cmd/virtualization-module-hooks/register.go @@ -22,6 +22,7 @@ import ( _ "hooks/pkg/hooks/discovery-clusterip-service-for-dvcr" _ "hooks/pkg/hooks/discovery-workload-nodes" _ "hooks/pkg/hooks/drop-openshift-labels" + _ "hooks/pkg/hooks/dvcr-maintenance" _ "hooks/pkg/hooks/generate-secret-for-dvcr" _ "hooks/pkg/hooks/migrate-delete-renamed-validation-admission-policy" _ "hooks/pkg/hooks/migrate-virthandler-kvm-node-labels" diff --git a/images/hooks/pkg/hooks/dvcr-maintenance/hook.go b/images/hooks/pkg/hooks/dvcr-maintenance/hook.go new file mode 100644 index 0000000000..7f48bc73ee --- /dev/null +++ b/images/hooks/pkg/hooks/dvcr-maintenance/hook.go @@ -0,0 +1,135 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dvcr_maintenance + +import ( + "context" + "fmt" + + "hooks/pkg/settings" + + "k8s.io/utils/ptr" + + "github.com/deckhouse/module-sdk/pkg" + "github.com/deckhouse/module-sdk/pkg/registry" +) + +/** +This hook watches over Secret/dvcr-maintenance in d8-virtualization namespace. + +If secret is present, hook sets value that switches DVCR Deployment in the maintenance mode. + +When the secret is gone, value is unset. +*/ + +const ( + SecretSnapshotName = "dvcr-maintenance-secret" + SecretJQFilter = `{ + "metadata": { + "name": .metadata.name, + "annotations": .metadata.annotations, + }, + }` + + secretName = "dvcr-maintenance" + dvcrMaintenancePath = "virtualization.internal.dvcr.maintenanceModeEnabled" + + dvcrDeploymentSwitchToMaintenanceModeAnno = "virtualization.deckhouse.io/dvcr-deployment-switch-to-maintenance-mode" +) + +type CASecret struct { + Crt []byte `json:"crt"` + Key []byte `json:"key"` +} + +var _ = registry.RegisterFunc(configDVCRMaintenance, handlerDVCRMaintenance) + +var configDVCRMaintenance = &pkg.HookConfig{ + OnBeforeHelm: &pkg.OrderedConfig{Order: 1}, + Kubernetes: []pkg.KubernetesConfig{ + { + Name: SecretSnapshotName, + APIVersion: "v1", + Kind: "Secret", + JqFilter: SecretJQFilter, + + ExecuteHookOnSynchronization: ptr.To(false), + + NameSelector: &pkg.NameSelector{ + MatchNames: []string{secretName}, + }, + + NamespaceSelector: &pkg.NamespaceSelector{ + NameSelector: &pkg.NameSelector{ + MatchNames: []string{settings.ModuleNamespace}, + }, + }, + }, + }, + + Queue: fmt.Sprintf("modules/%s", settings.ModuleName), +} + +func handlerDVCRMaintenance(_ context.Context, input *pkg.HookInput) error { + secretSnaps := input.Snapshots.Get(SecretSnapshotName) + secrets, err := parseSecretSnapshot(secretSnaps) + if err != nil { + return err + } + + input.Values.Set(dvcrMaintenancePath, isMaintenanceEnabled(secrets)) + return nil +} + +func isMaintenanceEnabled(secrets []partialSecret) string { + if len(secrets) == 0 { + return "false" + } + if _, ok := secrets[0].Metadata.Annotations[dvcrDeploymentSwitchToMaintenanceModeAnno]; ok { + return "true" + } + + return "false" +} + +type partialSecret struct { + Metadata partialSecretMetadata `json:"metadata"` +} + +type partialSecretMetadata struct { + Name string `json:"name"` + Annotations map[string]string `json:"annotations"` +} + +func parseSecretSnapshot(snaps []pkg.Snapshot) ([]partialSecret, error) { + secrets := make([]partialSecret, 0, len(snaps)) + + if len(snaps) == 0 { + return secrets, nil + } + + for _, snap := range snaps { + var secret partialSecret + err := snap.UnmarshalTo(&secret) + if err != nil { + return nil, err + } + secrets = append(secrets, secret) + } + + return secrets, nil +} diff --git a/images/virtualization-artifact/cmd/virtualization-controller/main.go b/images/virtualization-artifact/cmd/virtualization-controller/main.go index f4b0f9d26a..10975690c1 100644 --- a/images/virtualization-artifact/cmd/virtualization-controller/main.go +++ b/images/virtualization-artifact/cmd/virtualization-controller/main.go @@ -41,6 +41,7 @@ import ( "github.com/deckhouse/deckhouse/pkg/log" appconfig "github.com/deckhouse/virtualization-controller/pkg/config" "github.com/deckhouse/virtualization-controller/pkg/controller/cvi" + dvcrmaintenance "github.com/deckhouse/virtualization-controller/pkg/controller/dvcr-maintenance" "github.com/deckhouse/virtualization-controller/pkg/controller/evacuation" "github.com/deckhouse/virtualization-controller/pkg/controller/indexer" "github.com/deckhouse/virtualization-controller/pkg/controller/livemigration" @@ -434,6 +435,12 @@ func main() { os.Exit(1) } + dvcrMaintenanceLogger := logger.NewControllerLogger(dvcrmaintenance.ControllerName, logLevel, logOutput, logDebugVerbosity, logDebugControllerList) + if _, err = dvcrmaintenance.NewController(ctx, mgr, dvcrMaintenanceLogger, dvcrSettings); err != nil { + log.Error(err.Error()) + os.Exit(1) + } + log.Info("Starting the Manager.") // Start the Manager diff --git a/images/virtualization-artifact/pkg/common/annotations/annotations.go b/images/virtualization-artifact/pkg/common/annotations/annotations.go index ba7453b2a8..710201f5bb 100644 --- a/images/virtualization-artifact/pkg/common/annotations/annotations.go +++ b/images/virtualization-artifact/pkg/common/annotations/annotations.go @@ -187,6 +187,9 @@ const ( AnnVMOPUID = AnnAPIGroupV + "/vmop-uid" // AnnVMOPSnapshotName is an annotation on vmop that represents name a snapshot created for VMOP. AnnVMOPSnapshotName = AnnAPIGroupV + "/vmop-snapshot-name" + + AnnDVCRDeploymentSwitchToMaintenanceMode = AnnAPIGroupV + "/dvcr-deployment-switch-to-maintenance-mode" + AnnDVCRGarbageCollectionDone = AnnAPIGroupV + "/dvcr-garbage-collection-done" ) // AddAnnotation adds an annotation to an object diff --git a/images/virtualization-artifact/pkg/config/load_dvcr_settings.go b/images/virtualization-artifact/pkg/config/load_dvcr_settings.go index b9b674f668..e53b94723c 100644 --- a/images/virtualization-artifact/pkg/config/load_dvcr_settings.go +++ b/images/virtualization-artifact/pkg/config/load_dvcr_settings.go @@ -36,6 +36,8 @@ const ( DVCRCertsSecretNSVar = "DVCR_CERTS_SECRET_NAMESPACE" // DVCRInsecureTLSVar is an env variable holds the flag whether DVCR is insecure. DVCRInsecureTLSVar = "DVCR_INSECURE_TLS" + // DVCRGCScheduleVar is an env variable holds the cron schedule to run DVCR garbage collection. + DVCRGCScheduleVar = "DVCR_GC_SCHEDULE" // UploaderIngressHostVar is a env variable UploaderIngressHostVar = "UPLOADER_INGRESS_HOST" @@ -55,6 +57,7 @@ func LoadDVCRSettingsFromEnvs(controllerNamespace string) (*dvcr.Settings, error CertsSecretNamespace: os.Getenv(DVCRCertsSecretNSVar), RegistryURL: os.Getenv(DVCRRegistryURLVar), InsecureTLS: os.Getenv(DVCRInsecureTLSVar), + GCSchedule: os.Getenv(DVCRGCScheduleVar), UploaderIngressSettings: dvcr.UploaderIngressSettings{ Host: os.Getenv(UploaderIngressHostVar), TLSSecret: os.Getenv(UploaderIngressTLSSecretVar), @@ -79,5 +82,9 @@ func LoadDVCRSettingsFromEnvs(controllerNamespace string) (*dvcr.Settings, error dvcrSettings.UploaderIngressSettings.TLSSecretNamespace = controllerNamespace } + if dvcrSettings.GCSchedule == "" { + dvcrSettings.GCSchedule = dvcr.DefaultGCSchedule + } + return dvcrSettings, nil } diff --git a/images/virtualization-artifact/pkg/controller/conditions/accessor.go b/images/virtualization-artifact/pkg/controller/conditions/accessor.go new file mode 100644 index 0000000000..7620268dcd --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/conditions/accessor.go @@ -0,0 +1,51 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package conditions + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/deckhouse/virtualization/api/core/v1alpha2" +) + +type ConditionsAccessor interface { + Conditions() *[]metav1.Condition +} + +type conditionsAccessorImpl struct { + conditions *[]metav1.Condition +} + +func (c *conditionsAccessorImpl) Conditions() *[]metav1.Condition { + return c.conditions +} + +func NewConditionsAccessor(obj client.Object) ConditionsAccessor { + var ptr *[]metav1.Condition + switch v := obj.(type) { + case *v1alpha2.ClusterVirtualImage: + ptr = &v.Status.Conditions + case *v1alpha2.VirtualImage: + ptr = &v.Status.Conditions + case *v1alpha2.VirtualDisk: + ptr = &v.Status.Conditions + case *v1alpha2.VirtualMachine: + ptr = &v.Status.Conditions + } + return &conditionsAccessorImpl{conditions: ptr} +} diff --git a/images/virtualization-artifact/pkg/controller/cvi/cvi_controller.go b/images/virtualization-artifact/pkg/controller/cvi/cvi_controller.go index 401fbad06f..bfbba9c794 100644 --- a/images/virtualization-artifact/pkg/controller/cvi/cvi_controller.go +++ b/images/virtualization-artifact/pkg/controller/cvi/cvi_controller.go @@ -30,6 +30,7 @@ import ( "github.com/deckhouse/deckhouse/pkg/log" "github.com/deckhouse/virtualization-controller/pkg/controller/cvi/internal" "github.com/deckhouse/virtualization-controller/pkg/controller/cvi/internal/source" + "github.com/deckhouse/virtualization-controller/pkg/controller/dvcr-maintenance/postponehandler" "github.com/deckhouse/virtualization-controller/pkg/controller/service" "github.com/deckhouse/virtualization-controller/pkg/dvcr" "github.com/deckhouse/virtualization-controller/pkg/eventrecord" @@ -56,24 +57,26 @@ func NewController( importerImage string, uploaderImage string, requirements corev1.ResourceRequirements, - dvcr *dvcr.Settings, + dvcrSettings *dvcr.Settings, ns string, ) (controller.Controller, error) { stat := service.NewStatService(log) protection := service.NewProtectionService(mgr.GetClient(), v1alpha2.FinalizerCVIProtection) - importer := service.NewImporterService(dvcr, mgr.GetClient(), importerImage, requirements, PodPullPolicy, PodVerbose, ControllerName, protection) - uploader := service.NewUploaderService(dvcr, mgr.GetClient(), uploaderImage, requirements, PodPullPolicy, PodVerbose, ControllerName, protection) - disk := service.NewDiskService(mgr.GetClient(), dvcr, protection, ControllerName) + importer := service.NewImporterService(dvcrSettings, mgr.GetClient(), importerImage, requirements, PodPullPolicy, PodVerbose, ControllerName, protection) + uploader := service.NewUploaderService(dvcrSettings, mgr.GetClient(), uploaderImage, requirements, PodPullPolicy, PodVerbose, ControllerName, protection) + disk := service.NewDiskService(mgr.GetClient(), dvcrSettings, protection, ControllerName) + dvcrService := service.NewDVCRService(mgr.GetClient()) recorder := eventrecord.NewEventRecorderLogger(mgr, ControllerName) sources := source.NewSources() - sources.Set(v1alpha2.DataSourceTypeHTTP, source.NewHTTPDataSource(recorder, stat, importer, dvcr, ns)) - sources.Set(v1alpha2.DataSourceTypeContainerImage, source.NewRegistryDataSource(recorder, stat, importer, dvcr, mgr.GetClient(), ns)) - sources.Set(v1alpha2.DataSourceTypeObjectRef, source.NewObjectRefDataSource(recorder, stat, importer, disk, dvcr, mgr.GetClient(), ns)) - sources.Set(v1alpha2.DataSourceTypeUpload, source.NewUploadDataSource(recorder, stat, uploader, dvcr, ns)) + sources.Set(v1alpha2.DataSourceTypeHTTP, source.NewHTTPDataSource(recorder, stat, importer, dvcrSettings, dvcrService, ns)) + sources.Set(v1alpha2.DataSourceTypeContainerImage, source.NewRegistryDataSource(recorder, stat, importer, dvcrSettings, mgr.GetClient(), ns)) + sources.Set(v1alpha2.DataSourceTypeObjectRef, source.NewObjectRefDataSource(recorder, stat, importer, disk, dvcrSettings, mgr.GetClient(), ns)) + sources.Set(v1alpha2.DataSourceTypeUpload, source.NewUploadDataSource(recorder, stat, uploader, dvcrSettings, ns)) reconciler := NewReconciler( mgr.GetClient(), + postponehandler.New[*v1alpha2.ClusterVirtualImage](dvcrService, recorder), internal.NewDatasourceReadyHandler(sources), internal.NewLifeCycleHandler(sources, mgr.GetClient()), internal.NewDeletionHandler(sources), diff --git a/images/virtualization-artifact/pkg/controller/cvi/internal/source/http.go b/images/virtualization-artifact/pkg/controller/cvi/internal/source/http.go index 5a8ba64cdf..ed02b9525e 100644 --- a/images/virtualization-artifact/pkg/controller/cvi/internal/source/http.go +++ b/images/virtualization-artifact/pkg/controller/cvi/internal/source/http.go @@ -46,6 +46,7 @@ type HTTPDataSource struct { statService Stat importerService Importer dvcrSettings *dvcr.Settings + dvcrService DVCRMaintenance controllerNamespace string recorder eventrecord.EventRecorderLogger } @@ -55,12 +56,14 @@ func NewHTTPDataSource( statService Stat, importerService Importer, dvcrSettings *dvcr.Settings, + dvcrService DVCRMaintenance, controllerNamespace string, ) *HTTPDataSource { return &HTTPDataSource{ statService: statService, importerService: importerService, dvcrSettings: dvcrSettings, + dvcrService: dvcrService, controllerNamespace: controllerNamespace, recorder: recorder, } diff --git a/images/virtualization-artifact/pkg/controller/cvi/internal/source/interfaces.go b/images/virtualization-artifact/pkg/controller/cvi/internal/source/interfaces.go index 09cc02d606..fd8eac5c2e 100644 --- a/images/virtualization-artifact/pkg/controller/cvi/internal/source/interfaces.go +++ b/images/virtualization-artifact/pkg/controller/cvi/internal/source/interfaces.go @@ -70,3 +70,8 @@ type Stat interface { IsUploadStarted(ownerUID types.UID, pod *corev1.Pod) bool CheckPod(pod *corev1.Pod) error } + +type DVCRMaintenance interface { + GetMaintenanceSecret(ctx context.Context) (*corev1.Secret, error) + IsMaintenanceInitiatedOrInProgress(secret *corev1.Secret) bool +} diff --git a/images/virtualization-artifact/pkg/controller/dvcr-maintenance/condition/deployment.go b/images/virtualization-artifact/pkg/controller/dvcr-maintenance/condition/deployment.go new file mode 100644 index 0000000000..6e2343becf --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/dvcr-maintenance/condition/deployment.go @@ -0,0 +1,91 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package condition + +import ( + "fmt" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + dvcr_deployment_condition "github.com/deckhouse/virtualization/api/core/v1alpha2/dvcr-deployment-condition" +) + +func NewMaintenanceCondition(reason dvcr_deployment_condition.MaintenanceReason, msgf string, args ...any) appsv1.DeploymentCondition { + status := "Unknown" + switch reason { + case dvcr_deployment_condition.LastResult: + status = "False" + case dvcr_deployment_condition.InProgress: + status = "True" + } + + return appsv1.DeploymentCondition{ + Type: dvcr_deployment_condition.MaintenanceType, + Status: corev1.ConditionStatus(status), + LastUpdateTime: metav1.Now(), + Reason: string(reason), + Message: fmt.Sprintf(msgf, args...), + } +} + +// UpdateMaintenanceCondition replaces or removes Maintenance condition from deployment status. +// Return true if status was changed. +func UpdateMaintenanceCondition(deploy *appsv1.Deployment, reason dvcr_deployment_condition.MaintenanceReason, msgf string, args ...any) { + if deploy == nil { + return + } + + condition := NewMaintenanceCondition(reason, msgf, args...) + + // Condition is nil, so remove maintenance condition. + if len(deploy.Status.Conditions) > 0 { + filteredConditions := make([]appsv1.DeploymentCondition, 0) + for _, cond := range deploy.Status.Conditions { + if cond.Type == dvcr_deployment_condition.MaintenanceType { + if cond.Reason != condition.Reason || cond.Message != condition.Message { + condition.LastTransitionTime = metav1.Now() + } + cond = condition + } + // Copy non-maintenance conditions. + filteredConditions = append(filteredConditions, cond) + } + deploy.Status.Conditions = filteredConditions + } + + // Deploy has no conditions, create new slice. + deploy.Status.Conditions = []appsv1.DeploymentCondition{condition} +} + +// DeleteMaintenanceCondition removes Maintenance condition from deployment status. +func DeleteMaintenanceCondition(deploy *appsv1.Deployment) { + if deploy == nil || len(deploy.Status.Conditions) == 0 { + return + } + + // Filter conditions to remove maintenance condition. + filteredConditions := make([]appsv1.DeploymentCondition, 0) + for _, cond := range deploy.Status.Conditions { + if cond.Type != dvcr_deployment_condition.MaintenanceType { + // Copy only non-maintenance conditions. + filteredConditions = append(filteredConditions, cond) + } + } + deploy.Status.Conditions = filteredConditions +} diff --git a/images/virtualization-artifact/pkg/controller/dvcr-maintenance/dvcr_maintenance_controller.go b/images/virtualization-artifact/pkg/controller/dvcr-maintenance/dvcr_maintenance_controller.go new file mode 100644 index 0000000000..52acc5e4ad --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/dvcr-maintenance/dvcr_maintenance_controller.go @@ -0,0 +1,85 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dvcrmaintenance + +import ( + "context" + "fmt" + "time" + + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/manager" + + "github.com/deckhouse/deckhouse/pkg/log" + "github.com/deckhouse/virtualization-controller/pkg/controller/dvcr-maintenance/internal" + internalservice "github.com/deckhouse/virtualization-controller/pkg/controller/dvcr-maintenance/internal/service" + dvcrtypes "github.com/deckhouse/virtualization-controller/pkg/controller/dvcr-maintenance/types" + "github.com/deckhouse/virtualization-controller/pkg/controller/gc" + "github.com/deckhouse/virtualization-controller/pkg/controller/service" + "github.com/deckhouse/virtualization-controller/pkg/dvcr" + "github.com/deckhouse/virtualization-controller/pkg/logger" +) + +const ( + ControllerName = "dvcr-maintenance-controller" +) + +func NewController( + ctx context.Context, + mgr manager.Manager, + log *log.Logger, + dvcrSettings *dvcr.Settings, +) (controller.Controller, error) { + // init services + dvcrService := service.NewDVCRService(mgr.GetClient()) + provisioningLister := internalservice.NewProvisioningLister(mgr.GetClient()) + + reconciler := NewReconciler( + mgr.GetClient(), + internal.NewLifeCycleHandler(mgr.GetClient(), dvcrService, provisioningLister), + ) + + dvcrController, err := controller.New(ControllerName, mgr, controller.Options{ + Reconciler: reconciler, + RecoverPanic: ptr.To(true), + LogConstructor: logger.NewConstructor(log), + CacheSyncTimeout: 10 * time.Minute, + }) + if err != nil { + return nil, err + } + + if err = reconciler.SetupController(ctx, mgr, dvcrController); err != nil { + return nil, err + } + + // Not an elegant solution, but it is easier to add cron watches here, than in internal/watcher package. + // Cron source to initiate garbage collection from the user specified schedule. + cronSourceGC, err := gc.NewCronSource(dvcrSettings.GCSchedule, gc.NewSingleObjectLister(dvcrtypes.CronSourceNamespace, dvcrtypes.CronSourceRunGC), log) + if err != nil { + return nil, fmt.Errorf("setup DVCR cleanup cron source: %w", err) + } + err = dvcrController.Watch(cronSourceGC) + if err != nil { + return nil, fmt.Errorf("failed to setup dvcr-cleanup cron watcher: %w", err) + } + + log.Info("Initialized DVCR maintenance controller") + + return dvcrController, nil +} diff --git a/images/virtualization-artifact/pkg/controller/dvcr-maintenance/dvcr_maintenance_reconciler.go b/images/virtualization-artifact/pkg/controller/dvcr-maintenance/dvcr_maintenance_reconciler.go new file mode 100644 index 0000000000..d153282c3c --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/dvcr-maintenance/dvcr_maintenance_reconciler.go @@ -0,0 +1,96 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dvcrmaintenance + +import ( + "context" + "fmt" + "reflect" + + appsv1 "k8s.io/api/apps/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/deckhouse/virtualization-controller/pkg/controller/dvcr-maintenance/internal/watcher" + dvcrtypes "github.com/deckhouse/virtualization-controller/pkg/controller/dvcr-maintenance/types" + "github.com/deckhouse/virtualization-controller/pkg/controller/reconciler" +) + +type Watcher interface { + Watch(mgr manager.Manager, ctr controller.Controller) error +} + +type Handler interface { + Handle(ctx context.Context, req reconcile.Request, deploy *appsv1.Deployment) (reconcile.Result, error) +} + +type Reconciler struct { + handlers []Handler + client client.Client +} + +func NewReconciler(client client.Client, handlers ...Handler) *Reconciler { + return &Reconciler{ + client: client, + handlers: handlers, + } +} + +func (r *Reconciler) Reconcile(ctx context.Context, req reconcile.Request) (reconcile.Result, error) { + deploy := reconciler.NewResource(dvcrtypes.DVCRDeploymentKey(), r.client, r.factory, r.statusGetter) + err := deploy.Fetch(ctx) + if err != nil { + return reconcile.Result{}, err + } + // DVCR maintenance is needless if Deploy/dvcr is absent. + if deploy.IsEmpty() { + return reconcile.Result{}, nil + } + + rec := reconciler.NewBaseReconciler[Handler](r.handlers) + rec.SetHandlerExecutor(func(ctx context.Context, h Handler) (reconcile.Result, error) { + return h.Handle(ctx, req, deploy.Changed()) + }) + rec.SetResourceUpdater(func(ctx context.Context) error { + return deploy.Update(ctx) + }) + + return rec.Reconcile(ctx) +} + +func (r *Reconciler) SetupController(_ context.Context, mgr manager.Manager, ctr controller.Controller) error { + for _, w := range []Watcher{ + watcher.NewDVCRMaintenanceSecretWatcher(mgr.GetClient()), + } { + err := w.Watch(mgr, ctr) + if err != nil { + return fmt.Errorf("failed to setup watcher %s: %w", reflect.TypeOf(w).Elem().Name(), err) + } + } + + return nil +} + +func (r *Reconciler) factory() *appsv1.Deployment { + return &appsv1.Deployment{} +} + +func (r *Reconciler) statusGetter(obj *appsv1.Deployment) appsv1.DeploymentStatus { + return obj.Status +} diff --git a/images/virtualization-artifact/pkg/controller/dvcr-maintenance/internal/cleanup.go b/images/virtualization-artifact/pkg/controller/dvcr-maintenance/internal/cleanup.go new file mode 100644 index 0000000000..3a5dc7b9cd --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/dvcr-maintenance/internal/cleanup.go @@ -0,0 +1,17 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package internal diff --git a/images/virtualization-artifact/pkg/controller/dvcr-maintenance/internal/life_cycle.go b/images/virtualization-artifact/pkg/controller/dvcr-maintenance/internal/life_cycle.go new file mode 100644 index 0000000000..fdff5a6707 --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/dvcr-maintenance/internal/life_cycle.go @@ -0,0 +1,119 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package internal + +import ( + "context" + "fmt" + "time" + + appsv1 "k8s.io/api/apps/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + dvcrcondition "github.com/deckhouse/virtualization-controller/pkg/controller/dvcr-maintenance/condition" + dvcrtypes "github.com/deckhouse/virtualization-controller/pkg/controller/dvcr-maintenance/types" + dvcr_deployment_condition "github.com/deckhouse/virtualization/api/core/v1alpha2/dvcr-deployment-condition" +) + +type LifeCycleHandler struct { + client client.Client + dvcrService dvcrtypes.DVCRService + provisioningLister dvcrtypes.ProvisioningLister +} + +func NewLifeCycleHandler(client client.Client, dvcrService dvcrtypes.DVCRService, provisioningLister dvcrtypes.ProvisioningLister) *LifeCycleHandler { + return &LifeCycleHandler{ + client: client, + dvcrService: dvcrService, + provisioningLister: provisioningLister, + } +} + +func (h LifeCycleHandler) Handle(ctx context.Context, req reconcile.Request, deploy *appsv1.Deployment) (reconcile.Result, error) { + if req.Namespace == dvcrtypes.CronSourceNamespace && req.Name == dvcrtypes.CronSourceRunGC { + dvcrcondition.UpdateMaintenanceCondition(deploy, + dvcr_deployment_condition.InProgress, + "garbage collection initiated", + ) + return reconcile.Result{}, h.dvcrService.InitiateMaintenanceMode(ctx) + } + + if req.Name == dvcrtypes.DVCRMaintenanceSecretName { + // Secret has 3 states: + // - created, without annotations (InitiatedNotStarted) + // - wait for all provisioners to finish, add switch annotation. + // - add condition to deployment + // - switch annotation is present. (Started) + // - wait for cleanup to finish, just return. + // - add condition to deployment + // - done annotation is present. (Done) + // - cleanup is done, copy result to deployment condition. + // - delete secret. + secret, err := h.dvcrService.GetMaintenanceSecret(ctx) + if err != nil { + return reconcile.Result{}, fmt.Errorf("fetch maintenance secret: %w", err) + } + if secret == nil || secret.GetDeletionTimestamp() != nil { + // Secret is gone, no action required. + return reconcile.Result{}, nil + } + + if h.dvcrService.IsMaintenanceDone(secret) { + dvcrcondition.UpdateMaintenanceCondition(deploy, + dvcr_deployment_condition.LastResult, + "%s", string(secret.Data["result"]), + ) + return reconcile.Result{}, h.dvcrService.DeleteMaintenanceSecret(ctx) + } + + if h.dvcrService.IsMaintenanceStarted(secret) { + dvcrcondition.UpdateMaintenanceCondition(deploy, + dvcr_deployment_condition.InProgress, + "wait for garbage collection to finish", + ) + // Wait for done annotation. + return reconcile.Result{}, nil + } + + // No special annotation, check for provisioners to finish. + resourcesInProvisioning, err := h.provisioningLister.ListAllInProvisioning(ctx) + if err != nil { + return reconcile.Result{}, fmt.Errorf("list resources in provisioning: %w", err) + } + remainInProvisioning := len(resourcesInProvisioning) + if remainInProvisioning > 0 { + dvcrcondition.UpdateMaintenanceCondition(deploy, + dvcr_deployment_condition.InProgress, + "wait for cvi/vi/vd finish provisioning: %d resources remain", remainInProvisioning, + ) + return reconcile.Result{RequeueAfter: time.Second * 20}, nil + } + // All provisioners are finished, switch to garbage collection. + err = h.dvcrService.SwitchToMaintenanceMode(ctx) + if err != nil { + return reconcile.Result{}, fmt.Errorf("switch to maintenance mode: %w", err) + } + dvcrcondition.UpdateMaintenanceCondition(deploy, + dvcr_deployment_condition.InProgress, + "wait for garbage collection to finish", + ) + return reconcile.Result{}, nil + } + + return reconcile.Result{}, nil +} diff --git a/images/virtualization-artifact/pkg/controller/dvcr-maintenance/internal/service/provisioning_lister.go b/images/virtualization-artifact/pkg/controller/dvcr-maintenance/internal/service/provisioning_lister.go new file mode 100644 index 0000000000..bfa20d05a4 --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/dvcr-maintenance/internal/service/provisioning_lister.go @@ -0,0 +1,163 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "context" + "fmt" + + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/deckhouse/virtualization-controller/pkg/controller/conditions" + "github.com/deckhouse/virtualization/api/core/v1alpha2" + "github.com/deckhouse/virtualization/api/core/v1alpha2/cvicondition" + "github.com/deckhouse/virtualization/api/core/v1alpha2/vdcondition" + "github.com/deckhouse/virtualization/api/core/v1alpha2/vicondition" +) + +type ProvisioningLister struct { + client client.Client +} + +func NewProvisioningLister(client client.Client) *ProvisioningLister { + return &ProvisioningLister{ + client: client, + } +} + +func (p *ProvisioningLister) ListAllInProvisioning(ctx context.Context) ([]client.Object, error) { + result := make([]client.Object, 0) + + clusterImages, err := p.ListClusterVirtualImagesInProvisioning(ctx) + if err != nil { + return nil, err + } + for i := range clusterImages { + result = append(result, &clusterImages[i]) + } + + images, err := p.ListVirtualImagesInProvisioning(ctx) + if err != nil { + return nil, err + } + for i := range images { + result = append(result, &images[i]) + } + + disks, err := p.ListVirtualDisksInProvisioning(ctx) + if err != nil { + return nil, err + } + for i := range disks { + result = append(result, &disks[i]) + } + + return result, nil +} + +func (p *ProvisioningLister) ListClusterVirtualImagesInProvisioning(ctx context.Context) ([]v1alpha2.ClusterVirtualImage, error) { + var cviList v1alpha2.ClusterVirtualImageList + err := p.client.List(ctx, &cviList) + if err != nil { + return nil, fmt.Errorf("list all ClusterVirtualImages: %w", err) + } + + provisioning := make([]v1alpha2.ClusterVirtualImage, 0) + + for _, cvi := range cviList.Items { + cond, ok := conditions.GetCondition(cvicondition.ReadyType, cvi.Status.Conditions) + if !ok { + continue + } + if cond.Status == "False" && cond.Reason == cvicondition.Provisioning.String() { + provisioning = append(provisioning, cvi) + } + } + + return provisioning, nil +} + +func (p *ProvisioningLister) ListVirtualImagesInProvisioning(ctx context.Context) ([]v1alpha2.VirtualImage, error) { + var viList v1alpha2.VirtualImageList + err := p.client.List(ctx, &viList) + if err != nil { + return nil, fmt.Errorf("list all VirtualImages: %w", err) + } + + provisioning := make([]v1alpha2.VirtualImage, 0) + + for _, vi := range viList.Items { + cond, ok := conditions.GetCondition(vicondition.ReadyType, vi.Status.Conditions) + if !ok { + continue + } + if cond.Status == "False" && cond.Reason == vicondition.Provisioning.String() { + provisioning = append(provisioning, vi) + } + } + + return provisioning, nil +} + +func (p *ProvisioningLister) ListVirtualDisksInProvisioning(ctx context.Context) ([]v1alpha2.VirtualDisk, error) { + var vdList v1alpha2.VirtualDiskList + err := p.client.List(ctx, &vdList) + if err != nil { + return nil, fmt.Errorf("list all VirtualDiks: %w", err) + } + + provisioning := make([]v1alpha2.VirtualDisk, 0) + + for _, vd := range vdList.Items { + // Ignore disks without "import to dvcr first" stage. + if !vdHasDVCRStage(&vd) { + continue + } + cond, ok := conditions.GetCondition(vdcondition.ReadyType, vd.Status.Conditions) + if !ok { + continue + } + if cond.Status == "False" && cond.Reason == vdcondition.Provisioning.String() { + provisioning = append(provisioning, vd) + } + } + + return provisioning, nil +} + +// vdHasDVCRStage returns true if ClusterVirtualImage, VirtualImage or VirtualDosk +// upload images into DVCR. +func vdHasDVCRStage(vd *v1alpha2.VirtualDisk) bool { + if vd == nil || vd.Spec.DataSource == nil { + return false + } + switch vd.Spec.DataSource.Type { + case v1alpha2.DataSourceTypeHTTP, + v1alpha2.DataSourceTypeContainerImage, + v1alpha2.DataSourceTypeUpload: + return true + } + if vd.Spec.DataSource.ObjectRef == nil { + return false + } + switch vd.Spec.DataSource.ObjectRef.Kind { + case v1alpha2.VirtualDiskObjectRefKindVirtualImage, + v1alpha2.VirtualDiskObjectRefKindClusterVirtualImage: + return true + } + return false +} diff --git a/images/virtualization-artifact/pkg/controller/dvcr-maintenance/internal/watcher/dvcr_deployment.go b/images/virtualization-artifact/pkg/controller/dvcr-maintenance/internal/watcher/dvcr_deployment.go new file mode 100644 index 0000000000..05d240b9bb --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/dvcr-maintenance/internal/watcher/dvcr_deployment.go @@ -0,0 +1,72 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package watcher + +import ( + "context" + + appsv1 "k8s.io/api/apps/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/controller-runtime/pkg/source" + + dvcrtypes "github.com/deckhouse/virtualization-controller/pkg/controller/dvcr-maintenance/types" +) + +type DVCRDeploymentWatcher struct { + client client.Client +} + +func NewDVCRDeploymentWatcher(client client.Client) *DVCRDeploymentWatcher { + return &DVCRDeploymentWatcher{ + client: client, + } +} + +// Watch adds watching for Deployment/dvcr changes and for cron events. +func (w *DVCRDeploymentWatcher) Watch(mgr manager.Manager, ctr controller.Controller) error { + if err := ctr.Watch( + source.Kind( + mgr.GetCache(), + &appsv1.Deployment{}, + handler.TypedEnqueueRequestsFromMapFunc(func(ctx context.Context, deploy *appsv1.Deployment) []reconcile.Request { + if deploy.GetNamespace() == dvcrtypes.ModuleNamespace && deploy.GetName() == dvcrtypes.DVCRDeploymentName { + return []reconcile.Request{ + { + NamespacedName: client.ObjectKeyFromObject(deploy), + }, + } + } + return nil + }), + predicate.TypedFuncs[*appsv1.Deployment]{ + UpdateFunc: func(e event.TypedUpdateEvent[*appsv1.Deployment]) bool { + return e.ObjectOld.GetGeneration() != e.ObjectNew.GetGeneration() + }, + }, + ), + ); err != nil { + return err + } + + return nil +} diff --git a/images/virtualization-artifact/pkg/controller/dvcr-maintenance/internal/watcher/maintenance_secret.go b/images/virtualization-artifact/pkg/controller/dvcr-maintenance/internal/watcher/maintenance_secret.go new file mode 100644 index 0000000000..c7ceddaa53 --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/dvcr-maintenance/internal/watcher/maintenance_secret.go @@ -0,0 +1,69 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package watcher + +import ( + "reflect" + + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/manager" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/source" + + dvcrtypes "github.com/deckhouse/virtualization-controller/pkg/controller/dvcr-maintenance/types" +) + +type DVCRMaintenanceSecretWatcher struct { + client client.Client +} + +func NewDVCRMaintenanceSecretWatcher(client client.Client) *DVCRMaintenanceSecretWatcher { + return &DVCRMaintenanceSecretWatcher{ + client: client, + } +} + +// Watch adds watching for Deployment/dvcr changes and for cron events. +func (w *DVCRMaintenanceSecretWatcher) Watch(mgr manager.Manager, ctr controller.Controller) error { + if err := ctr.Watch( + source.Kind( + mgr.GetCache(), + &corev1.Secret{}, + &handler.TypedEnqueueRequestForObject[*corev1.Secret]{}, + predicate.TypedFuncs[*corev1.Secret]{ + UpdateFunc: func(e event.TypedUpdateEvent[*corev1.Secret]) bool { + // Handle only dvcr-maintenance secret. + if e.ObjectOld.Namespace != dvcrtypes.ModuleNamespace { + return false + } + if e.ObjectOld.Name != dvcrtypes.DVCRMaintenanceSecretName { + return false + } + return !reflect.DeepEqual(e.ObjectNew.GetAnnotations(), e.ObjectOld.GetAnnotations()) + }, + }, + ), + ); err != nil { + return err + } + + return nil +} diff --git a/images/virtualization-artifact/pkg/controller/dvcr-maintenance/postponehandler/postpone.go b/images/virtualization-artifact/pkg/controller/dvcr-maintenance/postponehandler/postpone.go new file mode 100644 index 0000000000..556eb0d7b3 --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/dvcr-maintenance/postponehandler/postpone.go @@ -0,0 +1,131 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package postponehandler + +import ( + "context" + "fmt" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/deckhouse/virtualization-controller/pkg/controller/conditions" + "github.com/deckhouse/virtualization-controller/pkg/controller/reconciler" + "github.com/deckhouse/virtualization-controller/pkg/eventrecord" + "github.com/deckhouse/virtualization/api/core/v1alpha2" + "github.com/deckhouse/virtualization/api/core/v1alpha2/cvicondition" + "github.com/deckhouse/virtualization/api/core/v1alpha2/vdcondition" + "github.com/deckhouse/virtualization/api/core/v1alpha2/vicondition" +) + +type DVCRService interface { + GetMaintenanceSecret(ctx context.Context) (*corev1.Secret, error) + IsMaintenanceInitiatedOrInProgress(*corev1.Secret) bool +} + +var PostponePeriod = time.Second * 15 + +type Postpone[object client.Object] struct { + dvcrService DVCRService + recorder eventrecord.EventRecorderLogger +} + +func New[T client.Object](dvcrService DVCRService, recorder eventrecord.EventRecorderLogger) *Postpone[T] { + return &Postpone[T]{ + dvcrService: dvcrService, + recorder: recorder, + } +} + +// Handle sets Ready condition to Provisioning for newly created resources +// if dvcr is in the maintenance mode. +// Applicable for ClusterVirtualImage, VirtualImage, and VirtualDisk. +func (p *Postpone[T]) Handle(ctx context.Context, obj T) (reconcile.Result, error) { + maintenanceSecret, err := p.dvcrService.GetMaintenanceSecret(ctx) + if err != nil { + return reconcile.Result{}, fmt.Errorf("checking DVCR maintenance mode: %w", err) + } + + conditionsPtr := conditions.NewConditionsAccessor(obj).Conditions() + // Only newly created resources are marked to postpone. + + readyCondition, exists := conditions.GetCondition(getReadyType(obj), *conditionsPtr) + isAlreadyPostponed := exists && readyCondition.Reason != ProvisioningPostponedReason.String() + isMaintenance := p.dvcrService.IsMaintenanceInitiatedOrInProgress(maintenanceSecret) + + // Clear PostponeProvisioning reason if maintenance finished. + if !isMaintenance { + if isAlreadyPostponed { + p.recorder.Event( + obj, + corev1.EventTypeNormal, + v1alpha2.ReasonImageOperationContinueAfterDVCRMaintenance, + "Continue image operation after finishing DVCR maintenance mode.", + ) + conditions.RemoveCondition(getReadyType(obj), conditionsPtr) + } + return reconcile.Result{}, nil + } + + // Postpone only newly created resources without Ready condition. + if !exists { + p.recorder.Event( + obj, + corev1.EventTypeNormal, + v1alpha2.ReasonImageOperationPostponedDueToDVCRMaintenance, + "Postpone image operation until the end of DVCR maintenance mode.", + ) + + // Set Provisioning to False. + cb := conditions.NewConditionBuilder(getReadyType(obj)).Generation(obj.GetGeneration()) + cb.Status(metav1.ConditionFalse). + Reason(ProvisioningPostponedReason). + Message("DVCR is in maintenance mode: wait until it finishes before creating provisioner.") + conditions.SetCondition(cb, conditions.NewConditionsAccessor(obj).Conditions()) + } + return reconcile.Result{RequeueAfter: PostponePeriod}, reconciler.ErrStopHandlerChain +} + +func (p *Postpone[T]) Name() string { + return "postpone-on-dvcr-maintenance-handler" +} + +func getReadyType(obj client.Object) conditions.Stringer { + switch obj.(type) { + case *v1alpha2.ClusterVirtualImage: + return cvicondition.ReadyType + case *v1alpha2.VirtualImage: + return vicondition.ReadyType + case *v1alpha2.VirtualDisk: + return vdcondition.ReadyType + } + + return stringer{str: "Ready"} +} + +type stringer struct { + str string +} + +func (s stringer) String() string { + return s.str +} + +var ProvisioningPostponedReason = stringer{str: "ProvisioningPostponed"} diff --git a/images/virtualization-artifact/pkg/controller/dvcr-maintenance/types/const.go b/images/virtualization-artifact/pkg/controller/dvcr-maintenance/types/const.go new file mode 100644 index 0000000000..221171bab5 --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/dvcr-maintenance/types/const.go @@ -0,0 +1,46 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +import "k8s.io/apimachinery/pkg/types" + +const ( + ModuleNamespace = "d8-virtualization" + + DVCRDeploymentName = "dvcr" + DVCRMaintenanceSecretName = "dvcr-maintenance" + CronSourceNamespace = "__cron_source__" + CronSourceRunGC = "run-gc" + CronSourceProvisioningPoll = "provisioning-poll" + + // ProvisioningPollSchedule is a cron schedule to poll provisioning status: every 20 seconds. + ProvisioningPollSchedule = "*/20 * * * * *" +) + +func DVCRDeploymentKey() types.NamespacedName { + return types.NamespacedName{ + Namespace: ModuleNamespace, + Name: DVCRDeploymentName, + } +} + +func DVCRMaintenanceSecretKey() types.NamespacedName { + return types.NamespacedName{ + Namespace: ModuleNamespace, + Name: DVCRMaintenanceSecretName, + } +} diff --git a/images/virtualization-artifact/pkg/controller/dvcr-maintenance/types/interfaces.go b/images/virtualization-artifact/pkg/controller/dvcr-maintenance/types/interfaces.go new file mode 100644 index 0000000000..2d1e3ab4ba --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/dvcr-maintenance/types/interfaces.go @@ -0,0 +1,41 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package types + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type DVCRService interface { + GetMaintenanceSecret(ctx context.Context) (*corev1.Secret, error) + DeleteMaintenanceSecret(ctx context.Context) error + InitiateMaintenanceMode(ctx context.Context) error + SwitchToMaintenanceMode(ctx context.Context) error + + // IsMaintenanceSecretExist(secret *corev1.Secret) bool + + IsMaintenanceInitiatedNotStarted(secret *corev1.Secret) bool + IsMaintenanceStarted(secret *corev1.Secret) bool + IsMaintenanceDone(secret *corev1.Secret) bool +} + +type ProvisioningLister interface { + ListAllInProvisioning(ctx context.Context) ([]client.Object, error) +} diff --git a/images/virtualization-artifact/pkg/controller/gc/cron_source.go b/images/virtualization-artifact/pkg/controller/gc/cron_source.go index 545fb19d31..7af950b5f2 100644 --- a/images/virtualization-artifact/pkg/controller/gc/cron_source.go +++ b/images/virtualization-artifact/pkg/controller/gc/cron_source.go @@ -51,9 +51,10 @@ var _ source.Source = &CronSource{} const sourceName = "CronSource" func NewCronSource(scheduleSpec string, objLister ObjectLister, log *log.Logger) (*CronSource, error) { - schedule, err := cron.ParseStandard(scheduleSpec) + specParser := cron.NewParser(cron.Second | cron.SecondOptional | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow) + schedule, err := specParser.Parse(scheduleSpec) if err != nil { - return nil, fmt.Errorf("parsing standard spec %q: %w", scheduleSpec, err) + return nil, fmt.Errorf("parse schedule: %w", err) } return &CronSource{ diff --git a/images/virtualization-artifact/pkg/controller/service/dvcr_service.go b/images/virtualization-artifact/pkg/controller/service/dvcr_service.go new file mode 100644 index 0000000000..d43c0bcc33 --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/service/dvcr_service.go @@ -0,0 +1,163 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package service + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/deckhouse/virtualization-controller/pkg/common/annotations" +) + +type DVCRService struct { + client client.Client +} + +func NewDVCRService(client client.Client) *DVCRService { + return &DVCRService{ + client: client, + } +} + +const ( + moduleNamespace = "d8-virtualization" + dvcrDeploymentName = "dvcr" + maintenanceModeSecretName = "dvcr-maintenance" +) + +func (d *DVCRService) CreateMaintenanceModeSecret(ctx context.Context) error { + secret := &corev1.Secret{ + TypeMeta: metav1.TypeMeta{ + Kind: "Secret", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Namespace: moduleNamespace, + Name: maintenanceModeSecretName, + }, + } + return d.client.Create(ctx, secret) +} + +// IsMaintenanceSecretExist returns true if maintenance secret exists. +func (d *DVCRService) IsMaintenanceSecretExist(ctx context.Context) (bool, error) { + secret, err := d.GetMaintenanceSecret(ctx) + return secret != nil, err +} + +// IsMaintenanceInitiatedNotStarted returns true if secret exists but +// cleanup is not done yet. +// Use it to postpone switch deployment to maintenance until all write operations are finished. +func (d *DVCRService) IsMaintenanceInitiatedNotStarted(secret *corev1.Secret) bool { + if secret == nil { + return false + } + _, switched := secret.GetAnnotations()[annotations.AnnDVCRDeploymentSwitchToMaintenanceMode] + _, done := secret.GetAnnotations()[annotations.AnnDVCRGarbageCollectionDone] + return !switched && !done +} + +// IsMaintenanceStarted returns true if switch to maintenance mode is on. +// Use it to determine "wait" state. +func (d *DVCRService) IsMaintenanceStarted(secret *corev1.Secret) bool { + if secret == nil { + return false + } + _, switched := secret.GetAnnotations()[annotations.AnnDVCRDeploymentSwitchToMaintenanceMode] + return switched +} + +// IsMaintenanceInitiatedOrInProgress returns true if secret exists but +// cleanup is not done yet. (Use it to postpone rw operations with registry). +func (d *DVCRService) IsMaintenanceInitiatedOrInProgress(secret *corev1.Secret) bool { + if secret == nil { + return false + } + _, done := secret.GetAnnotations()[annotations.AnnDVCRGarbageCollectionDone] + return !done +} + +// IsMaintenanceDone returns true if secret exists and annotated with +// "done" annotation. +func (d *DVCRService) IsMaintenanceDone(secret *corev1.Secret) bool { + if secret == nil { + return false + } + _, done := secret.GetAnnotations()[annotations.AnnDVCRGarbageCollectionDone] + return done +} + +func (d *DVCRService) InitiateMaintenanceMode(ctx context.Context) error { + secret, err := d.GetMaintenanceSecret(ctx) + if err != nil { + return fmt.Errorf("get maintenance secret: %w", err) + } + if secret == nil { + return d.CreateMaintenanceModeSecret(ctx) + } + + // Update existing secret to initial state: remove annotations and data. + secret.SetAnnotations(nil) + secret.Data = nil + return d.client.Update(ctx, secret) +} + +func (d *DVCRService) SwitchToMaintenanceMode(ctx context.Context) error { + secret, err := d.GetMaintenanceSecret(ctx) + if secret == nil { + return fmt.Errorf("get maintenance secret to update: %w", err) + } + + objAnnotations := secret.GetAnnotations() + if objAnnotations == nil { + objAnnotations = make(map[string]string) + } + objAnnotations[annotations.AnnDVCRDeploymentSwitchToMaintenanceMode] = "" + secret.SetAnnotations(objAnnotations) + return d.client.Update(ctx, secret) +} + +func (d *DVCRService) GetMaintenanceSecret(ctx context.Context) (*corev1.Secret, error) { + var secret corev1.Secret + secretKey := types.NamespacedName{ + Namespace: moduleNamespace, + Name: maintenanceModeSecretName, + } + err := d.client.Get(ctx, secretKey, &secret) + if err != nil { + if k8serrors.IsNotFound(err) { + return nil, nil + } + return nil, err + } + + return &secret, nil +} + +func (d *DVCRService) DeleteMaintenanceSecret(ctx context.Context) error { + secret := &corev1.Secret{} + secret.SetNamespace(moduleNamespace) + secret.SetName(maintenanceModeSecretName) + err := d.client.Delete(ctx, secret) + return client.IgnoreNotFound(err) +} diff --git a/images/virtualization-artifact/pkg/controller/vd/internal/postpone_filter.go b/images/virtualization-artifact/pkg/controller/vd/internal/postpone_filter.go new file mode 100644 index 0000000000..0c8c6e5a80 --- /dev/null +++ b/images/virtualization-artifact/pkg/controller/vd/internal/postpone_filter.go @@ -0,0 +1,75 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package internal + +import ( + "context" + "fmt" + + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + "github.com/deckhouse/virtualization-controller/pkg/controller/dvcr-maintenance/postponehandler" + "github.com/deckhouse/virtualization-controller/pkg/logger" + "github.com/deckhouse/virtualization/api/core/v1alpha2" +) + +const postponeFilterHandlerPrefix = "vd-" + +type PostponeHandlerPreFilter struct { + postponeHandler *postponehandler.Postpone[*v1alpha2.VirtualDisk] +} + +// NewPostponeHandlerPreFilter runs postpone handler only if VirtualDisk is required to import/upload +// to DVCR first. +func NewPostponeHandlerPreFilter(postponeHandler *postponehandler.Postpone[*v1alpha2.VirtualDisk]) *PostponeHandlerPreFilter { + return &PostponeHandlerPreFilter{ + postponeHandler: postponeHandler, + } +} + +func (h PostponeHandlerPreFilter) Handle(ctx context.Context, vd *v1alpha2.VirtualDisk) (reconcile.Result, error) { + log := logger.FromContext(ctx).With(logger.SlogHandler(postponeFilterHandlerPrefix + h.postponeHandler.Name())) + + shouldRunDVCRClient, err := h.shouldRunDVCRClient(vd) + if err != nil { + return reconcile.Result{}, err + } + + if !shouldRunDVCRClient { + log.Debug("Ignore running handler to postpone on DVCR maintenance") + return reconcile.Result{}, nil + } + + return h.postponeHandler.Handle(logger.ToContext(ctx, log), vd) +} + +func (h PostponeHandlerPreFilter) shouldRunDVCRClient(vd *v1alpha2.VirtualDisk) (bool, error) { + if vd == nil || vd.Spec.DataSource == nil { + return false, nil + } + + switch vd.Spec.DataSource.Type { + case v1alpha2.DataSourceTypeHTTP, + v1alpha2.DataSourceTypeUpload, + v1alpha2.DataSourceTypeContainerImage: + return true, nil + case v1alpha2.DataSourceTypeObjectRef: + return false, nil + } + + return false, fmt.Errorf("unknown dataSource.type %s", vd.Spec.DataSource.Type) +} diff --git a/images/virtualization-artifact/pkg/controller/vd/vd_controller.go b/images/virtualization-artifact/pkg/controller/vd/vd_controller.go index 87d5aa2793..3343088c31 100644 --- a/images/virtualization-artifact/pkg/controller/vd/vd_controller.go +++ b/images/virtualization-artifact/pkg/controller/vd/vd_controller.go @@ -29,6 +29,7 @@ import ( "github.com/deckhouse/deckhouse/pkg/log" "github.com/deckhouse/virtualization-controller/pkg/config" + "github.com/deckhouse/virtualization-controller/pkg/controller/dvcr-maintenance/postponehandler" "github.com/deckhouse/virtualization-controller/pkg/controller/service" "github.com/deckhouse/virtualization-controller/pkg/controller/vd/internal" intsvc "github.com/deckhouse/virtualization-controller/pkg/controller/vd/internal/service" @@ -68,6 +69,7 @@ func NewController( uploader := service.NewUploaderService(dvcr, mgr.GetClient(), uploaderImage, requirements, PodPullPolicy, PodVerbose, ControllerName, protection) disk := service.NewDiskService(mgr.GetClient(), dvcr, protection, ControllerName) scService := intsvc.NewVirtualDiskStorageClassService(service.NewBaseStorageClassService(mgr.GetClient()), storageClassSettings) + dvcrService := service.NewDVCRService(mgr.GetClient()) recorder := eventrecord.NewEventRecorderLogger(mgr, ControllerName) blank := source.NewBlankDataSource(recorder, disk, mgr.GetClient()) @@ -80,6 +82,7 @@ func NewController( reconciler := NewReconciler( mgr.GetClient(), + internal.NewPostponeHandlerPreFilter(postponehandler.New[*v1alpha2.VirtualDisk](dvcrService, recorder)), internal.NewInitHandler(), internal.NewStorageClassReadyHandler(scService), internal.NewDatasourceReadyHandler(recorder, blank, sources), diff --git a/images/virtualization-artifact/pkg/controller/vi/vi_controller.go b/images/virtualization-artifact/pkg/controller/vi/vi_controller.go index 534f07598e..709c8e9494 100644 --- a/images/virtualization-artifact/pkg/controller/vi/vi_controller.go +++ b/images/virtualization-artifact/pkg/controller/vi/vi_controller.go @@ -29,6 +29,7 @@ import ( "github.com/deckhouse/deckhouse/pkg/log" "github.com/deckhouse/virtualization-controller/pkg/config" + "github.com/deckhouse/virtualization-controller/pkg/controller/dvcr-maintenance/postponehandler" "github.com/deckhouse/virtualization-controller/pkg/controller/service" "github.com/deckhouse/virtualization-controller/pkg/controller/vi/internal" intsvc "github.com/deckhouse/virtualization-controller/pkg/controller/vi/internal/service" @@ -69,6 +70,7 @@ func NewController( bounder := service.NewBounderPodService(dvcr, mgr.GetClient(), bounderImage, requirements, PodPullPolicy, PodVerbose, ControllerName, protection) disk := service.NewDiskService(mgr.GetClient(), dvcr, protection, ControllerName) scService := intsvc.NewVirtualImageStorageClassService(service.NewBaseStorageClassService(mgr.GetClient()), storageClassSettings) + dvcrService := service.NewDVCRService(mgr.GetClient()) recorder := eventrecord.NewEventRecorderLogger(mgr, ControllerName) sources := source.NewSources() @@ -79,6 +81,7 @@ func NewController( reconciler := NewReconciler( mgr.GetClient(), + postponehandler.New[*v1alpha2.VirtualImage](dvcrService, recorder), internal.NewStorageClassReadyHandler(recorder, scService), internal.NewDatasourceReadyHandler(sources), internal.NewLifeCycleHandler(recorder, sources, mgr.GetClient()), diff --git a/images/virtualization-artifact/pkg/dvcr/dvcr.go b/images/virtualization-artifact/pkg/dvcr/dvcr.go index 503df7c242..6b9bef766e 100644 --- a/images/virtualization-artifact/pkg/dvcr/dvcr.go +++ b/images/virtualization-artifact/pkg/dvcr/dvcr.go @@ -38,6 +38,8 @@ type Settings struct { InsecureTLS string // UploaderIngressSettings are settings for uploading images to the DVCR using ingress. UploaderIngressSettings UploaderIngressSettings + // GCSchedule is a cron formatted schedule to periodically run a garbage collection. + GCSchedule string } type UploaderIngressSettings struct { @@ -48,9 +50,10 @@ type UploaderIngressSettings struct { } const ( - CVMIImageTmpl = "cvi/%s:%s" - VMIImageTmpl = "vi/%s/%s:%s" - VMDImageTmpl = "vd/%s/%s:%s" + CVMIImageTmpl = "cvi/%s:%s" + VMIImageTmpl = "vi/%s/%s:%s" + VMDImageTmpl = "vd/%s/%s:%s" + DefaultGCSchedule = "0 2 * * *" ) // RegistryImageForCVI returns image name for CVI. diff --git a/openapi/config-values.yaml b/openapi/config-values.yaml index 5cf5baae25..5dbd852e6d 100644 --- a/openapi/config-values.yaml +++ b/openapi/config-values.yaml @@ -179,6 +179,18 @@ properties: properties: type: enum: ["ObjectStorage"] + gc: + description: | + Parameters for garbage collection. + type: object + properties: + schedule: + type: string + default: "0 2 * * *" + description: | + Schedule to run garbage collection procedure that remove stale images for `ClusterVirtualImage`, `VirtualImage`, `VirtualDisk` resources deleted from the cluster. + + Default is to run every day at 2:00 AM ("0 2 * * *"). audit: type: object description: | diff --git a/openapi/doc-ru-config-values.yaml b/openapi/doc-ru-config-values.yaml index 041db60763..7a28c942c3 100644 --- a/openapi/doc-ru-config-values.yaml +++ b/openapi/doc-ru-config-values.yaml @@ -92,6 +92,11 @@ properties: bucket: description: | Контейнер, в котором вы можете хранить свои файлы и объекты данных. + autoCleanupSchedule: + description: | + Расписание запуска процедуры авто-очистки, которая удалит образы для ресурсов `ClusterVirtualImage`, `VirtualImage`, `VirtualDisk`, которые были удалены из кластера. + + По умолчанию авто-очистка запускается каждый день в 2 часа ночи. ("0 2 * * *"). virtualImages: type: object description: | diff --git a/openapi/values.yaml b/openapi/values.yaml index a4da0203c7..f2111d62ab 100644 --- a/openapi/values.yaml +++ b/openapi/values.yaml @@ -39,6 +39,11 @@ properties: type: string serviceIP: type: string + maintenanceModeEnabled: + type: string + maintenanceMode: + type: string + default: "" controller: type: object default: {} diff --git a/templates/dvcr/_helpers.tpl b/templates/dvcr/_helpers.tpl index ceb42a103a..5a2b19ad53 100644 --- a/templates/dvcr/_helpers.tpl +++ b/templates/dvcr/_helpers.tpl @@ -4,6 +4,10 @@ true {{- end }} {{- end }} +{{- define "dvcr.isMaintenance" -}} +{{- .Values.virtualization.internal | dig "dvcr" "maintenanceModeEnabled" "false" | default "false" -}} +{{- end }} + {{- define "dvcr.envs" -}} - name: REGISTRY_HTTP_TLS_CERTIFICATE value: /etc/ssl/docker/tls.crt @@ -48,6 +52,13 @@ true {{- end }} {{- end }} +{{- define "dvcr.envs.maintenance" -}} +{{- if eq .Values.virtualization.internal.moduleConfig.dvcr.storage.type "PersistentVolumeClaim" }} +- name: REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY + value: "/var/lib/registry" +{{- end }} +{{- end }} + {{- define "dvcr.volumeMounts" -}} - name: "dvcr-config" @@ -68,6 +79,15 @@ true {{- end -}} +{{- define "dvcr.volumeMounts.maintenance" -}} +- name: "dvcr-config" + mountPath: "/etc/docker/registry" +{{- if eq .Values.virtualization.internal.moduleConfig.dvcr.storage.type "PersistentVolumeClaim" }} +- name: data + mountPath: /var/lib/registry/ +{{- end }} +{{- end -}} + {{- define "dvcr.volumes" -}} - name: dvcr-config @@ -94,7 +114,11 @@ true {{- define "dvcr.helm_lib_deployment_strategy_and_replicas_for_ha" -}} -{{- if and (include "helm_lib_ha_enabled" .) (eq .Values.virtualization.internal.moduleConfig.dvcr.storage.type "ObjectStorage") }} +{{- if eq (include "dvcr.isMaintenance" . ) "true" }} +replicas: 1 +strategy: + type: Recreate +{{- else if and (include "helm_lib_ha_enabled" .) (eq .Values.virtualization.internal.moduleConfig.dvcr.storage.type "ObjectStorage") }} replicas: 2 strategy: type: RollingUpdate @@ -135,3 +159,5 @@ strategy: {{- $context := index . 0 -}} {{- printf "dvcr.d8-%s.svc" $context.Chart.Name }} {{- end -}} + + diff --git a/templates/dvcr/configmap-cleanup.yaml b/templates/dvcr/configmap-cleanup.yaml new file mode 100644 index 0000000000..0e5dee3c37 --- /dev/null +++ b/templates/dvcr/configmap-cleanup.yaml @@ -0,0 +1,31 @@ +{{- if eq (include "dvcr.isEnabled" . ) "true"}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: dvcr-config-cleanup + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list . (dict "app" "dvcr" )) | nindent 2 }} +data: + config.yml: |- + version: 0.1 + log: + fields: + service: dvcr + storage: + cache: + blobdescriptor: inmemory + http: + addr: :5000 + headers: + X-Content-Type-Options: [nosniff] + debug: + addr: 127.0.0.1:5001 + prometheus: + enabled: true + path: /metrics + health: + storagedriver: + enabled: true + interval: 10s + threshold: 3 +{{- end }} diff --git a/templates/dvcr/configmap.yaml b/templates/dvcr/configmap.yaml index 1a71d3ab08..c53deeb5e4 100644 --- a/templates/dvcr/configmap.yaml +++ b/templates/dvcr/configmap.yaml @@ -14,6 +14,11 @@ data: storage: cache: blobdescriptor: inmemory +{{- if eq (include "dvcr.isMaintenance" . ) "true" }} + maintenance: + readonly: + enabled: true +{{- end }} http: addr: :5000 headers: diff --git a/templates/dvcr/deployment.yaml b/templates/dvcr/deployment.yaml index ebd1f73f95..18b9f205c3 100644 --- a/templates/dvcr/deployment.yaml +++ b/templates/dvcr/deployment.yaml @@ -1,5 +1,10 @@ {{- $priorityClassName := include "priorityClassName" . }} -{{- define "dvcr_resources" }} +{{- define "dvcr.resources" }} +cpu: 50m +memory: 15Mi +{{- end }} + +{{- define "dvcr.resources.maintenance" }} cpu: 50m memory: 15Mi {{- end }} @@ -25,7 +30,13 @@ spec: {{- include "kube_rbac_proxy.vpa_container_policy" . | nindent 4 }} - containerName: dvcr minAllowed: - {{- include "dvcr_resources" . | nindent 8 }} + {{- include "dvcr.resources" . | nindent 8 }} + maxAllowed: + cpu: 100m + memory: 250Mi + - containerName: dvcr-maintenance + minAllowed: + {{- include "dvcr.resources.maintenance" . | nindent 8 }} maxAllowed: cpu: 100m memory: 250Mi @@ -94,7 +105,7 @@ spec: requests: {{- include "helm_lib_module_ephemeral_storage_only_logs" . | nindent 14 }} {{- if not ( .Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} - {{- include "dvcr_resources" . | nindent 14 }} + {{- include "dvcr.resources" . | nindent 14 }} {{- end }} env: {{ include "dvcr.envs" . | nindent 12 }} volumeMounts: {{ include "dvcr.volumeMounts" . | nindent 12 }} @@ -105,6 +116,28 @@ spec: (dict "upstream" "http://127.0.0.1:5001/metrics" "path" "/metrics" "name" "dvcr") ) }} {{- include "kube_rbac_proxy.sidecar_container" (tuple . $kubeRbacProxySettings) | nindent 8 }} + + {{- if eq (include "dvcr.isMaintenance" . ) "true" }} + - name: dvcr-maintenance + image: {{ include "helm_lib_module_image" (list . "dvcr") }} + imagePullPolicy: IfNotPresent + command: + - /usr/local/bin/dvcr-cleaner + - gc + - auto-cleanup + - --maintenance-secret-name + - dvcr-maintenance + - --gc-timeout + - 10m + resources: + requests: + {{- include "helm_lib_module_ephemeral_storage_only_logs" . | nindent 14 }} + {{- if not ( .Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} + {{- include "dvcr.resources.maintenance" . | nindent 14 }} + {{- end }} + env: {{ include "dvcr.envs.maintenance" . | nindent 12 }} + volumeMounts: {{ include "dvcr.volumeMounts.maintenance" . | nindent 12 }} + {{- end }} volumes: {{ include "dvcr.volumes" . | nindent 8 }} {{- include "helm_lib_priority_class" (tuple . $priorityClassName) | nindent 6 }} {{- include "helm_lib_node_selector" (tuple . "system") | nindent 6 }} diff --git a/templates/dvcr/rbac-for-us.yaml b/templates/dvcr/rbac-for-us.yaml index 8fc3878957..1c5e78247d 100644 --- a/templates/dvcr/rbac-for-us.yaml +++ b/templates/dvcr/rbac-for-us.yaml @@ -8,6 +8,62 @@ metadata: namespace: d8-{{ .Chart.Name }} imagePullSecrets: - name: virtualization-module-registry + +# dvcr-maintenance +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: d8:virtualization:dvcr + {{- include "helm_lib_module_labels" (list . (dict "app" "dvcr")) | nindent 2 }} +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - update +- apiGroups: + - "apps" + resources: + - deployments + verbs: + - get + - list +- apiGroups: + - "apps" + resources: + - deployments/status + verbs: + - get + - list + - update + - patch +- apiGroups: + - virtualization.deckhouse.io + resources: + - virtualdisks + - virtualimages + - clustervirtualimages + verbs: + - get + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: d8:virtualization:dvcr + {{- include "helm_lib_module_labels" (list . (dict "app" "dvcr")) | nindent 2 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: d8:virtualization:dvcr +subjects: + - kind: ServiceAccount + name: dvcr + namespace: d8-{{ .Chart.Name }} --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/templates/virtualization-controller/_helpers.tpl b/templates/virtualization-controller/_helpers.tpl index 764e0d2df8..60943c38b9 100644 --- a/templates/virtualization-controller/_helpers.tpl +++ b/templates/virtualization-controller/_helpers.tpl @@ -36,6 +36,8 @@ true value: {{ $registry | quote }} - name: DVCR_INSECURE_TLS value: "true" +- name: DVCR_GC_SCHEDULE + value: "{{ .Values.virtualization.internal.moduleConfig | dig "dvcr" "gc" "schedule" "" }}" - name: VIRTUAL_MACHINE_CIDRS value: {{ join "," .Values.virtualization.internal.moduleConfig.virtualMachineCIDRs | quote }} {{- if (hasKey .Values.virtualization.internal.moduleConfig "virtualImages") }} diff --git a/templates/virtualization-controller/rbac-for-us.yaml b/templates/virtualization-controller/rbac-for-us.yaml index c80363ec94..4366708ef3 100644 --- a/templates/virtualization-controller/rbac-for-us.yaml +++ b/templates/virtualization-controller/rbac-for-us.yaml @@ -241,6 +241,13 @@ rules: - get - list - watch +- apiGroups: + - apps + resources: + - deployments/status + verbs: + - update + - patch - apiGroups: - "" resources: diff --git a/tools/kubeconform/fixtures/module-values.yaml b/tools/kubeconform/fixtures/module-values.yaml index c817ddfd6b..c857d7f949 100644 --- a/tools/kubeconform/fixtures/module-values.yaml +++ b/tools/kubeconform/fixtures/module-values.yaml @@ -376,6 +376,7 @@ virtualization: passwordRW: DVCR-PASSWD-STRING salt: DVCR-SALT-STRING serviceIP: 10.222.229.246 + maintenanceMode: "true" rootCA: ca: "" crt: ROOT-CA-CRT