Update pkg/utils/utils.go

Co-authored-by: Abdullah Gharaibeh <[email protected]>

Author: wangshulei098

docs/examples/sample/gang-scheduling/README.md

@@ -75,5 +75,10 @@ This ensures that resources for all pods are reserved before the leader starts,
 
 To assign all groups of a LeaderWorkerSet to a specific Volcano queue, add the `scheduling.volcano.sh/queue-name` annotation to the LeaderWorkerSet metadata.
 
+
+### Volcano Scheduler Configurations
+
+To configure certain annotations in the PodGroup that are used by the Volcano scheduler to control specific scheduling behaviors, the PodGroup created by the LeaderWorkerSet will inherit the annotations starting with `volcano.sh/` from the LeaderWorkerSet's metadata.annotations.
+
 For a complete example, please refer to the [lws-sample-volcano.yaml](./lws-sample-volcano.yaml) file.
 

docs/examples/sample/gang-scheduling/lws-sample-volcano.yaml

@@ -3,6 +3,7 @@ kind: LeaderWorkerSet
 metadata:
   # annotations:
   #   scheduling.volcano.sh/queue-name: lws-queue # Uncomment the annotation to set the queue name if needed
+  #   volcano.sh/sla-waiting-time: 5m  # PodGroups can inherit annotations with the prefix volcano.sh/. 
   name: lws-sample-volcano
 spec:
   replicas: 2

pkg/schedulerprovider/volcano_provider.go

@@ -18,6 +18,7 @@ package schedulerprovider
 
 import (
 	"context"
+	"strings"
 
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -31,7 +32,8 @@ import (
 )
 
 const (
-	Volcano ProviderType = "volcano"
+	Volcano                 ProviderType = "volcano"
+	VolcanoAnnotationPrefix string       = "volcano.sh/"
 )
 
 type VolcanoProvider struct {
@@ -63,7 +65,7 @@ func (v *VolcanoProvider) CreatePodGroupIfNotExists(ctx context.Context, lws *le
 					leaderworkerset.GroupIndexLabelKey: leaderPod.Labels[leaderworkerset.GroupIndexLabelKey],
 					leaderworkerset.RevisionKey:        leaderPod.Labels[leaderworkerset.RevisionKey],
 				},
-				Annotations: utils.InheritVolcanoAnnotations(lws),
+				Annotations: inheritVolcanoAnnotations(lws),
 			},
 			Spec: volcanov1beta1.PodGroupSpec{
 				// Default startupPolicy is LeaderCreated, Leader and Workers Pods are scheduled together, so MinAvailable is set to size
@@ -105,3 +107,13 @@ func (v *VolcanoProvider) InjectPodGroupMetadata(pod *corev1.Pod) error {
 
 	return nil
 }
+
+func inheritVolcanoAnnotations(lws *leaderworkerset.LeaderWorkerSet) map[string]string {
+	res := map[string]string{}
+	for k, v := range lws.Annotations {
+		if strings.HasPrefix(k, VolcanoAnnotationPrefix) {
+			res[k] = v
+		}
+	}
+	return res
+}

pkg/schedulerprovider/volcano_provider_test.go

@@ -50,6 +50,7 @@ func TestVolcanoProvider_CreatePodGroupIfNotExists(t *testing.T) {
 	testLeaderPod1 := createTestLeaderPod("test-lws-0", "default", "test-lws", "0", "abc123")
 	testLeaderPod2 := createTestLeaderPod("test-lws-1", "default", "test-lws", "1", "def456")
 	testLeaderPod3 := createTestLeaderPod("test-lws-0", "default", "test-lws", "0", "xyz789")
+	testLeaderPod4 := createTestLeaderPod("test-lws-0", "default", "test-lws", "0", "jkl012")
 
 	tests := []struct {
 		name           string
@@ -155,6 +156,52 @@ func TestVolcanoProvider_CreatePodGroupIfNotExists(t *testing.T) {
 				Spec:       volcanov1beta1.PodGroupSpec{MinMember: 3},
 			},
 		},
+		{
+			name: "create podgroup inherit volcano annotations",
+			lws: &leaderworkerset.LeaderWorkerSet{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-lws",
+					Namespace: "default",
+					Annotations: map[string]string{
+						"volcano.sh/sla-waiting-time": "5m",
+					},
+				},
+				Spec: leaderworkerset.LeaderWorkerSetSpec{
+					LeaderWorkerTemplate: leaderworkerset.LeaderWorkerTemplate{
+						Size: ptr.To[int32](3),
+						WorkerTemplate: corev1.PodTemplateSpec{
+							Spec: corev1.PodSpec{
+								Containers: []corev1.Container{{
+									Name: "worker", Image: "nginx",
+									Resources: corev1.ResourceRequirements{Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100m")}},
+								}},
+							},
+						},
+					},
+				},
+			},
+			leaderPod:   testLeaderPod4,
+			expectError: false,
+			expectedPG: &volcanov1beta1.PodGroup{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "test-lws-0-jkl012",
+					Namespace: "default",
+					Labels: map[string]string{
+						leaderworkerset.GroupIndexLabelKey: "0",
+						leaderworkerset.SetNameLabelKey:    "test-lws",
+						leaderworkerset.RevisionKey:        "jkl012",
+					},
+					Annotations: map[string]string{
+						"volcano.sh/sla-waiting-time": "5m",
+					},
+					OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(testLeaderPod4, corev1.SchemeGroupVersion.WithKind("Pod"))},
+				},
+				Spec: volcanov1beta1.PodGroupSpec{
+					MinMember:    3,
+					MinResources: &corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("300m")},
+				},
+			},
+		},
 		{
 			name: "generic error on getting podgroup",
 			lws: &leaderworkerset.LeaderWorkerSet{

pkg/utils/utils.go

@@ -33,7 +33,6 @@ import (
 
 const (
 	defaultNamespace = "lws-system"
-	AnnotationPrefix = "volcano.sh/"
 )
 
 // Sha1Hash accepts an input string and returns the 40 character SHA1 hash digest of the input string.
@@ -102,13 +101,3 @@ func CalculatePGMinResources(lws *leaderworkerset.LeaderWorkerSet) corev1.Resour
 
 	return totalResources
 }
-
-func InheritVolcanoAnnotations(lws *leaderworkerset.LeaderWorkerSet) map[string]string {
-	res := map[string]string{}
-	for k, v := range lws.Annotations {
-		if strings.HasPrefix(k, AnnotationPrefix) {
-			res[k] = v
-		}
-	}
-	return res
-}