Skip to content

Commit 70a8aa3

Browse files
tmonty12tmonty12
authored
feat(operator): mechanism for disabling imagePullSecrets discovery (#3092)
Signed-off-by: tmontfort <[email protected]>
1 parent 6675bfc commit 70a8aa3

File tree

4 files changed

+186
-2
lines changed

4 files changed

+186
-2
lines changed

‎deploy/cloud/operator/internal/consts/consts.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ const (
3030

3131
KubeAnnotationEnableGrove = "nvidia.com/enable-grove"
3232

33+
KubeAnnotationDisableImagePullSecretDiscovery = "nvidia.com/disable-image-pull-secret-discovery"
34+
3335
KubeLabelDynamoGraphDeploymentName = "nvidia.com/dynamo-graph-deployment-name"
3436
KubeLabelDynamoComponent = "nvidia.com/dynamo-component"
3537
KubeLabelDynamoNamespace = "nvidia.com/dynamo-namespace"

‎deploy/cloud/operator/internal/dynamo/graph.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -765,8 +765,10 @@ func GenerateBasePodSpec(
765765
maps.Copy(container.Resources.Limits, overrideResources.Limits)
766766
}
767767

768+
shouldDisableImagePullSecret := component.Annotations[commonconsts.KubeAnnotationDisableImagePullSecretDiscovery] == commonconsts.KubeLabelValueTrue
769+
768770
imagePullSecrets := []corev1.LocalObjectReference{}
769-
if secretsRetriever != nil && component.ExtraPodSpec != nil && component.ExtraPodSpec.MainContainer != nil && component.ExtraPodSpec.MainContainer.Image != "" {
771+
if !shouldDisableImagePullSecret && secretsRetriever != nil && component.ExtraPodSpec != nil && component.ExtraPodSpec.MainContainer != nil && component.ExtraPodSpec.MainContainer.Image != "" {
770772
secretsName, err := secretsRetriever.GetSecrets(namespace, component.ExtraPodSpec.MainContainer.Image)
771773
if err == nil {
772774
for _, secretName := range secretsName {

‎deploy/cloud/operator/internal/dynamo/graph_test.go‎

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3141,6 +3141,20 @@ func (m *mockSecretsRetriever) GetSecrets(namespace, registry string) ([]string,
31413141
return []string{}, nil
31423142
}
31433143

3144+
// Mock SecretsRetriever that returns secrets for testing docker secrets functionality
3145+
type mockSecretsRetrieverWithSecrets struct{}
3146+
3147+
func (m *mockSecretsRetrieverWithSecrets) RetrieveImagePullSecrets(ctx context.Context, deployment *v1alpha1.DynamoGraphDeployment) ([]corev1.LocalObjectReference, error) {
3148+
return []corev1.LocalObjectReference{
3149+
{Name: "test-docker-secret"},
3150+
}, nil
3151+
}
3152+
3153+
func (m *mockSecretsRetrieverWithSecrets) GetSecrets(namespace, registry string) ([]string, error) {
3154+
// Return some mock secrets when called
3155+
return []string{"test-docker-secret"}, nil
3156+
}
3157+
31443158
func TestGeneratePodSpecForComponent_SGLang(t *testing.T) {
31453159
secretsRetriever := &mockSecretsRetriever{}
31463160
dynamoDeployment := &v1alpha1.DynamoGraphDeployment{
@@ -4484,6 +4498,138 @@ func TestGenerateBasePodSpec_PlannerServiceAccount(t *testing.T) {
44844498
}
44854499
}
44864500

4501+
func TestGenerateBasePodSpec_DisableImagePullSecretDiscovery(t *testing.T) {
4502+
tests := []struct {
4503+
name string
4504+
component *v1alpha1.DynamoComponentDeploymentOverridesSpec
4505+
secretsRetriever SecretsRetriever
4506+
expectedImagePullSecrets []corev1.LocalObjectReference
4507+
}{
4508+
{
4509+
name: "disable docker secrets annotation set to true",
4510+
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
4511+
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
4512+
ComponentType: commonconsts.ComponentTypeFrontend,
4513+
Annotations: map[string]string{
4514+
commonconsts.KubeAnnotationDisableImagePullSecretDiscovery: commonconsts.KubeLabelValueTrue,
4515+
},
4516+
ExtraPodSpec: &common.ExtraPodSpec{
4517+
MainContainer: &corev1.Container{
4518+
Image: "test-registry/test-image:latest",
4519+
},
4520+
},
4521+
},
4522+
},
4523+
secretsRetriever: &mockSecretsRetrieverWithSecrets{},
4524+
expectedImagePullSecrets: nil, // Should be nil when disabled
4525+
},
4526+
{
4527+
name: "disable docker secrets annotation set to false",
4528+
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
4529+
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
4530+
ComponentType: commonconsts.ComponentTypeFrontend,
4531+
Annotations: map[string]string{
4532+
commonconsts.KubeAnnotationDisableImagePullSecretDiscovery: commonconsts.KubeLabelValueFalse,
4533+
},
4534+
ExtraPodSpec: &common.ExtraPodSpec{
4535+
MainContainer: &corev1.Container{
4536+
Image: "test-registry/test-image:latest",
4537+
},
4538+
},
4539+
},
4540+
},
4541+
secretsRetriever: &mockSecretsRetrieverWithSecrets{},
4542+
expectedImagePullSecrets: []corev1.LocalObjectReference{
4543+
{Name: "test-docker-secret"},
4544+
}, // Should be present when enabled
4545+
},
4546+
{
4547+
name: "disable docker secrets annotation not set (default behavior)",
4548+
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
4549+
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
4550+
ComponentType: commonconsts.ComponentTypeFrontend,
4551+
ExtraPodSpec: &common.ExtraPodSpec{
4552+
MainContainer: &corev1.Container{
4553+
Image: "test-registry/test-image:latest",
4554+
},
4555+
},
4556+
},
4557+
},
4558+
secretsRetriever: &mockSecretsRetrieverWithSecrets{},
4559+
expectedImagePullSecrets: []corev1.LocalObjectReference{
4560+
{Name: "test-docker-secret"},
4561+
}, // Should be present by default
4562+
},
4563+
{
4564+
name: "disable docker secrets annotation set to invalid value",
4565+
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
4566+
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
4567+
ComponentType: commonconsts.ComponentTypeFrontend,
4568+
Annotations: map[string]string{
4569+
commonconsts.KubeAnnotationDisableImagePullSecretDiscovery: "invalid",
4570+
},
4571+
ExtraPodSpec: &common.ExtraPodSpec{
4572+
MainContainer: &corev1.Container{
4573+
Image: "test-registry/test-image:latest",
4574+
},
4575+
},
4576+
},
4577+
},
4578+
secretsRetriever: &mockSecretsRetrieverWithSecrets{},
4579+
expectedImagePullSecrets: []corev1.LocalObjectReference{
4580+
{Name: "test-docker-secret"},
4581+
}, // Should be present when annotation is not "true"
4582+
},
4583+
{
4584+
name: "disable docker secrets but no secrets retriever",
4585+
component: &v1alpha1.DynamoComponentDeploymentOverridesSpec{
4586+
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
4587+
ComponentType: commonconsts.ComponentTypeFrontend,
4588+
Annotations: map[string]string{
4589+
commonconsts.KubeAnnotationDisableImagePullSecretDiscovery: commonconsts.KubeLabelValueFalse,
4590+
},
4591+
ExtraPodSpec: &common.ExtraPodSpec{
4592+
MainContainer: &corev1.Container{
4593+
Image: "test-registry/test-image:latest",
4594+
},
4595+
},
4596+
},
4597+
},
4598+
secretsRetriever: nil,
4599+
expectedImagePullSecrets: nil, // Should be nil when no retriever
4600+
},
4601+
}
4602+
4603+
for _, tt := range tests {
4604+
t.Run(tt.name, func(t *testing.T) {
4605+
controllerConfig := controller_common.Config{}
4606+
4607+
podSpec, err := GenerateBasePodSpec(
4608+
tt.component,
4609+
BackendFrameworkNoop,
4610+
tt.secretsRetriever,
4611+
"test-deployment",
4612+
"default",
4613+
RoleMain,
4614+
1,
4615+
controllerConfig,
4616+
commonconsts.MultinodeDeploymentTypeGrove,
4617+
"test-service",
4618+
)
4619+
4620+
if err != nil {
4621+
t.Errorf("GenerateBasePodSpec() error = %v", err)
4622+
return
4623+
}
4624+
4625+
if !reflect.DeepEqual(podSpec.ImagePullSecrets, tt.expectedImagePullSecrets) {
4626+
t.Errorf("GenerateBasePodSpec() ImagePullSecrets = %v, want %v",
4627+
podSpec.ImagePullSecrets, tt.expectedImagePullSecrets)
4628+
}
4629+
})
4630+
}
4631+
}
4632+
44874633
func TestGenerateBasePodSpec_Worker(t *testing.T) {
44884634
secretsRetriever := &mockSecretsRetriever{}
44894635
controllerConfig := controller_common.Config{}

‎docs/guides/dynamo_deploy/create_deployment.md‎

Lines changed: 35 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,4 +130,38 @@ If you are a Dynamo contributor the [dynamo run guide](../dynamo_run.md) for det
130130
```yaml
131131
args:
132132
- --is-prefill-worker # For disaggregated prefill workers
133-
```
133+
```
134+
135+
### Image Pull Secret Configuration
136+
137+
#### Automatic Discovery and Injection
138+
139+
By default, the Dynamo operator automatically discovers and injects image pull secrets based on container registry host matching. The operator scans Docker config secrets within the same namespace and matches their registry hostnames to the container image URLs, automatically injecting the appropriate secrets into the pod's `imagePullSecrets`.
140+
141+
**Disabling Automatic Discovery:**
142+
To disable this behavior for a component and manually control image pull secrets:
143+
144+
```yaml
145+
YourWorker:
146+
dynamoNamespace: your-namespace
147+
componentType: worker
148+
annotations:
149+
nvidia.com/disable-image-pull-secret-discovery: "true"
150+
```
151+
152+
When disabled, you can manually specify secrets as you would for a normal pod spec via:
153+
```yaml
154+
YourWorker:
155+
dynamoNamespace: your-namespace
156+
componentType: worker
157+
annotations:
158+
nvidia.com/disable-image-pull-secret-discovery: "true"
159+
extraPodSpec:
160+
imagePullSecrets:
161+
- name: my-registry-secret
162+
- name: another-secret
163+
mainContainer:
164+
image: your-image
165+
```
166+
167+
This automatic discovery eliminates the need to manually configure image pull secrets for each deployment.

0 commit comments

Comments
 (0)