From c877cfafe1c037834d26935226cdc4b3e7847c43 Mon Sep 17 00:00:00 2001 From: Yi Chen Date: Fri, 30 May 2025 10:17:48 +0800 Subject: [PATCH 1/8] Adjust the indentation of yaml templates Signed-off-by: Yi Chen --- .../templates/deployment.yaml | 193 +++++++++--------- .../kuberay-operator/templates/service.yaml | 8 +- 2 files changed, 100 insertions(+), 101 deletions(-) diff --git a/helm-chart/kuberay-operator/templates/deployment.yaml b/helm-chart/kuberay-operator/templates/deployment.yaml index e64d95caf79..ec5fb1d7185 100644 --- a/helm-chart/kuberay-operator/templates/deployment.yaml +++ b/helm-chart/kuberay-operator/templates/deployment.yaml @@ -30,112 +30,111 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} spec: - {{- with .Values.imagePullSecrets }} + {{- with .Values.imagePullSecrets }} imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} + {{- toYaml . | nindent 6 }} + {{- end }} serviceAccountName: {{ include "kuberay-operator.serviceAccount.name" . }} {{- if and (.Values.logging.baseDir) (.Values.logging.fileName) }} volumes: - - name: kuberay-logs - {{- if .Values.logging.sizeLimit }} - emptyDir: - sizeLimit: {{ .Values.logging.sizeLimit }} - {{- else }} - emptyDir: {} - {{- end }} + - name: kuberay-logs + {{- if .Values.logging.sizeLimit }} + emptyDir: + sizeLimit: {{ .Values.logging.sizeLimit }} + {{- else }} + emptyDir: {} + {{- end }} {{- end }} {{- with .Values.podSecurityContext }} securityContext: {{- toYaml . | nindent 8 }} {{- end }} containers: - - name: {{ .Chart.Name }} - {{- with .Values.securityContext }} - securityContext: - {{- toYaml . | nindent 12 }} - {{- end }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" - {{- with .Values.image.pullPolicy }} - imagePullPolicy: {{ . }} - {{- end }} - {{- if and (.Values.logging.baseDir) (.Values.logging.fileName) }} - volumeMounts: - - name: kuberay-logs - mountPath: "{{ .Values.logging.baseDir }}" - {{- end }} - command: - - {{ .Values.operatorCommand }} - args: - {{- $argList := list -}} - {{- $argList = append $argList (include "kuberay.featureGates" . | trim) -}} - {{- if .Values.batchScheduler -}} - {{- if .Values.batchScheduler.enabled -}} - {{- $argList = append $argList "--enable-batch-scheduler" -}} - {{- end -}} - {{- if .Values.batchScheduler.name -}} - {{- $argList = append $argList (printf "--batch-scheduler=%s" .Values.batchScheduler.name) -}} - {{- end -}} - {{- end -}} - {{- $watchNamespace := "" -}} - {{- if and .Values.singleNamespaceInstall (not .Values.watchNamespace) -}} - {{- $watchNamespace = .Release.Namespace -}} - {{- else if .Values.watchNamespace -}} - {{- $watchNamespace = join "," .Values.watchNamespace -}} - {{- end -}} - {{- if $watchNamespace -}} - {{- $argList = append $argList "--watch-namespace" -}} - {{- $argList = append $argList $watchNamespace -}} - {{- end -}} - {{- if and (.Values.logging.baseDir) (.Values.logging.fileName) -}} - {{- $argList = append $argList "--log-file-path" -}} - {{- $argList = append $argList (printf "%s/%s" .Values.logging.baseDir .Values.logging.fileName) -}} - {{- end -}} - {{- if .Values.logging.stdoutEncoder -}} - {{- $argList = append $argList "--log-stdout-encoder" -}} - {{- $argList = append $argList .Values.logging.stdoutEncoder -}} - {{- end -}} - {{- if .Values.logging.fileEncoder -}} - {{- $argList = append $argList "--log-file-encoder" -}} - {{- $argList = append $argList .Values.logging.fileEncoder -}} - {{- end -}} - {{- if hasKey .Values "useKubernetesProxy" -}} - {{- $argList = append $argList (printf "--use-kubernetes-proxy=%t" .Values.useKubernetesProxy) -}} - {{- end -}} - {{- if hasKey .Values "leaderElectionEnabled" -}} - {{- $argList = append $argList (printf "--enable-leader-election=%t" .Values.leaderElectionEnabled) -}} - {{- end -}} - {{- if and (hasKey .Values "metrics") (hasKey .Values.metrics "enabled") }} - {{- $argList = append $argList (printf "--enable-metrics=%t" .Values.metrics.enabled) -}} - {{- end -}} - {{- (printf "\n") -}} - {{- $argList | toYaml | indent 12 }} - ports: - - name: http - containerPort: 8080 - protocol: TCP - {{- with .Values.env }} - env: - {{- toYaml . | nindent 12 }} - {{- end }} - livenessProbe: - httpGet: - path: /metrics - port: http - initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} - periodSeconds: {{ .Values.livenessProbe.periodSeconds }} - failureThreshold: {{ .Values.livenessProbe.failureThreshold }} - readinessProbe: - httpGet: - path: /metrics - port: http - initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} - periodSeconds: {{ .Values.readinessProbe.periodSeconds }} - failureThreshold: {{ .Values.readinessProbe.failureThreshold }} - {{- with .Values.resources }} - resources: - {{- toYaml . | nindent 12 }} - {{- end }} + - name: {{ .Chart.Name }} + {{- with .Values.securityContext }} + securityContext: + {{- toYaml . | nindent 10 }} + {{- end }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + {{- with .Values.image.pullPolicy }} + imagePullPolicy: {{ . }} + {{- end }} + {{- if and (.Values.logging.baseDir) (.Values.logging.fileName) }} + volumeMounts: + - name: kuberay-logs + mountPath: "{{ .Values.logging.baseDir }}" + {{- end }} + command: + - {{ .Values.operatorCommand }} + args: + {{- $argList := list -}} + {{- $argList = append $argList (include "kuberay.featureGates" . | trim) -}} + {{- if .Values.batchScheduler -}} + {{- if .Values.batchScheduler.enabled -}} + {{- $argList = append $argList "--enable-batch-scheduler" -}} + {{- end -}} + {{- if .Values.batchScheduler.name -}} + {{- $argList = append $argList (printf "--batch-scheduler=%s" .Values.batchScheduler.name) -}} + {{- end -}} + {{- end -}} + {{- $watchNamespace := "" -}} + {{- if and .Values.singleNamespaceInstall (not .Values.watchNamespace) -}} + {{- $watchNamespace = .Release.Namespace -}} + {{- else if .Values.watchNamespace -}} + {{- $watchNamespace = join "," .Values.watchNamespace -}} + {{- end -}} + {{- if $watchNamespace -}} + {{- $argList = append $argList "--watch-namespace" -}} + {{- $argList = append $argList $watchNamespace -}} + {{- end -}} + {{- if and (.Values.logging.baseDir) (.Values.logging.fileName) -}} + {{- $argList = append $argList "--log-file-path" -}} + {{- $argList = append $argList (printf "%s/%s" .Values.logging.baseDir .Values.logging.fileName) -}} + {{- end -}} + {{- if .Values.logging.stdoutEncoder -}} + {{- $argList = append $argList "--log-stdout-encoder" -}} + {{- $argList = append $argList .Values.logging.stdoutEncoder -}} + {{- end -}} + {{- if .Values.logging.fileEncoder -}} + {{- $argList = append $argList "--log-file-encoder" -}} + {{- $argList = append $argList .Values.logging.fileEncoder -}} + {{- end -}} + {{- if hasKey .Values "useKubernetesProxy" -}} + {{- $argList = append $argList (printf "--use-kubernetes-proxy=%t" .Values.useKubernetesProxy) -}} + {{- end -}} + {{- if hasKey .Values "leaderElectionEnabled" -}} + {{- $argList = append $argList (printf "--enable-leader-election=%t" .Values.leaderElectionEnabled) -}} + {{- end -}} + {{- if and (hasKey .Values "metrics") (hasKey .Values.metrics "enabled") }} + {{- $argList = append $argList (printf "--enable-metrics=%t" .Values.metrics.enabled) -}} + {{- end -}} + {{- $argList | toYaml | nindent 8 }} + ports: + - name: http + containerPort: 8080 + protocol: TCP + {{- with .Values.env }} + env: + {{- toYaml . | nindent 8 }} + {{- end }} + livenessProbe: + httpGet: + path: /metrics + port: http + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + readinessProbe: + httpGet: + path: /metrics + port: http + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 10 }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} @@ -146,5 +145,5 @@ spec: {{- end }} {{- with .Values.tolerations }} tolerations: - {{- toYaml . | nindent 8 }} + {{- toYaml . | nindent 6 }} {{- end }} diff --git a/helm-chart/kuberay-operator/templates/service.yaml b/helm-chart/kuberay-operator/templates/service.yaml index 725f31d19af..d240602f600 100644 --- a/helm-chart/kuberay-operator/templates/service.yaml +++ b/helm-chart/kuberay-operator/templates/service.yaml @@ -8,10 +8,10 @@ metadata: spec: type: {{ .Values.service.type }} ports: - - port: {{ .Values.service.port }} - targetPort: http - protocol: TCP - name: http + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http selector: app.kubernetes.io/name: {{ include "kuberay-operator.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} From 74f4f17239877cfd04a05fff4b9ec6569709874f Mon Sep 17 00:00:00 2001 From: Yi Chen Date: Fri, 30 May 2025 10:24:39 +0800 Subject: [PATCH 2/8] Adjust the order of some fields Signed-off-by: Yi Chen --- .../templates/deployment.yaml | 66 +++++++++---------- .../templates/rolebinding.yaml | 8 +-- 2 files changed, 37 insertions(+), 37 deletions(-) diff --git a/helm-chart/kuberay-operator/templates/deployment.yaml b/helm-chart/kuberay-operator/templates/deployment.yaml index ec5fb1d7185..eaea906e97e 100644 --- a/helm-chart/kuberay-operator/templates/deployment.yaml +++ b/helm-chart/kuberay-operator/templates/deployment.yaml @@ -30,40 +30,12 @@ spec: {{- toYaml . | nindent 8 }} {{- end }} spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 6 }} - {{- end }} - serviceAccountName: {{ include "kuberay-operator.serviceAccount.name" . }} - {{- if and (.Values.logging.baseDir) (.Values.logging.fileName) }} - volumes: - - name: kuberay-logs - {{- if .Values.logging.sizeLimit }} - emptyDir: - sizeLimit: {{ .Values.logging.sizeLimit }} - {{- else }} - emptyDir: {} - {{- end }} - {{- end }} - {{- with .Values.podSecurityContext }} - securityContext: - {{- toYaml . | nindent 8 }} - {{- end }} containers: - name: {{ .Chart.Name }} - {{- with .Values.securityContext }} - securityContext: - {{- toYaml . | nindent 10 }} - {{- end }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" {{- with .Values.image.pullPolicy }} imagePullPolicy: {{ . }} {{- end }} - {{- if and (.Values.logging.baseDir) (.Values.logging.fileName) }} - volumeMounts: - - name: kuberay-logs - mountPath: "{{ .Values.logging.baseDir }}" - {{- end }} command: - {{ .Values.operatorCommand }} args: @@ -109,13 +81,22 @@ spec: {{- $argList = append $argList (printf "--enable-metrics=%t" .Values.metrics.enabled) -}} {{- end -}} {{- $argList | toYaml | nindent 8 }} + {{- with .Values.env }} + env: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- if and (.Values.logging.baseDir) (.Values.logging.fileName) }} + volumeMounts: + - name: kuberay-logs + mountPath: "{{ .Values.logging.baseDir }}" + {{- end }} ports: - name: http containerPort: 8080 protocol: TCP - {{- with .Values.env }} - env: - {{- toYaml . | nindent 8 }} + {{- with .Values.resources }} + resources: + {{- toYaml . | nindent 10 }} {{- end }} livenessProbe: httpGet: @@ -131,10 +112,24 @@ spec: initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} periodSeconds: {{ .Values.readinessProbe.periodSeconds }} failureThreshold: {{ .Values.readinessProbe.failureThreshold }} - {{- with .Values.resources }} - resources: + {{- with .Values.securityContext }} + securityContext: {{- toYaml . | nindent 10 }} {{- end }} + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 6 }} + {{- end }} + {{- if and (.Values.logging.baseDir) (.Values.logging.fileName) }} + volumes: + - name: kuberay-logs + {{- if .Values.logging.sizeLimit }} + emptyDir: + sizeLimit: {{ .Values.logging.sizeLimit }} + {{- else }} + emptyDir: {} + {{- end }} + {{- end }} {{- with .Values.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} @@ -147,3 +142,8 @@ spec: tolerations: {{- toYaml . | nindent 6 }} {{- end }} + serviceAccountName: {{ include "kuberay-operator.serviceAccount.name" . }} + {{- with .Values.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/helm-chart/kuberay-operator/templates/rolebinding.yaml b/helm-chart/kuberay-operator/templates/rolebinding.yaml index f9dd4cd2fbb..b793de17201 100644 --- a/helm-chart/kuberay-operator/templates/rolebinding.yaml +++ b/helm-chart/kuberay-operator/templates/rolebinding.yaml @@ -5,12 +5,12 @@ metadata: name: {{ include "kuberay-operator.clusterRoleBinding.name" . }} labels: {{- include "kuberay-operator.labels" . | nindent 4 }} -subjects: -- kind: ServiceAccount - name: {{ include "kuberay-operator.serviceAccount.name" . }} - namespace: {{ .Release.Namespace }} roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: {{ include "kuberay-operator.clusterRole.name" . }} +subjects: +- kind: ServiceAccount + name: {{ include "kuberay-operator.serviceAccount.name" . }} + namespace: {{ .Release.Namespace }} {{- end }} From a67e3cf7c9b2a927579d9476574e5c3715d6af92 Mon Sep 17 00:00:00 2001 From: Yi Chen Date: Fri, 30 May 2025 10:30:36 +0800 Subject: [PATCH 3/8] Define helper template for kuberay operator image Signed-off-by: Yi Chen --- helm-chart/kuberay-operator/templates/_helpers.tpl | 7 +++++++ helm-chart/kuberay-operator/templates/deployment.yaml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/helm-chart/kuberay-operator/templates/_helpers.tpl b/helm-chart/kuberay-operator/templates/_helpers.tpl index aa48c4c0729..5ba127d656b 100644 --- a/helm-chart/kuberay-operator/templates/_helpers.tpl +++ b/helm-chart/kuberay-operator/templates/_helpers.tpl @@ -56,6 +56,13 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} {{- include "kuberay-operator.fullname" . }} {{- end -}} +{{- /* Create the name of the image to use. */ -}} +{{- define "kuberay-operator.image" -}} +{{- $imageRepository := .Values.image.repository | default "kuberay/operator" }} +{{- $imageTag := .Values.image.tag | default "latest" }} +{{- printf "%s:%s" $imageRepository $imageTag }} +{{- end -}} + {{/* FeatureGates */}} diff --git a/helm-chart/kuberay-operator/templates/deployment.yaml b/helm-chart/kuberay-operator/templates/deployment.yaml index eaea906e97e..2df45291e53 100644 --- a/helm-chart/kuberay-operator/templates/deployment.yaml +++ b/helm-chart/kuberay-operator/templates/deployment.yaml @@ -32,7 +32,7 @@ spec: spec: containers: - name: {{ .Chart.Name }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + image: {{ include "kuberay-operator.image" . }} {{- with .Values.image.pullPolicy }} imagePullPolicy: {{ . }} {{- end }} From 9f4dc2940070d44f0e7cd919fe23c0aedcad64be Mon Sep 17 00:00:00 2001 From: Yi Chen Date: Fri, 30 May 2025 10:32:20 +0800 Subject: [PATCH 4/8] Add Helm unittest for kuberay operator image Signed-off-by: Yi Chen --- helm-chart/kuberay-operator/tests/deployment_test.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/helm-chart/kuberay-operator/tests/deployment_test.yaml b/helm-chart/kuberay-operator/tests/deployment_test.yaml index d5fc4861fef..2c11fe15e4d 100644 --- a/helm-chart/kuberay-operator/tests/deployment_test.yaml +++ b/helm-chart/kuberay-operator/tests/deployment_test.yaml @@ -55,6 +55,16 @@ tests: path: spec.template.metadata.annotations.key2 value: value2 + - it: Should use the specified image if specified + set: + image: + repository: test-image + tag: test-tag + asserts: + - equal: + path: spec.template.spec.containers[*].image + value: test-image:test-tag + - it: Should use the specified image pull policy if `image.pullPolicy` is set set: image: From 7a5b4f4d4bb120d00fb9cc67d836a1609252b184 Mon Sep 17 00:00:00 2001 From: Yi Chen Date: Fri, 30 May 2025 11:33:14 +0800 Subject: [PATCH 5/8] Refactor kuberay operator args Signed-off-by: Yi Chen --- .../kuberay-operator/templates/_helpers.tpl | 14 ---- .../templates/deployment.yaml | 76 +++++++++---------- helm-chart/kuberay-operator/values.yaml | 14 ++-- 3 files changed, 41 insertions(+), 63 deletions(-) diff --git a/helm-chart/kuberay-operator/templates/_helpers.tpl b/helm-chart/kuberay-operator/templates/_helpers.tpl index 5ba127d656b..4026dc8550f 100644 --- a/helm-chart/kuberay-operator/templates/_helpers.tpl +++ b/helm-chart/kuberay-operator/templates/_helpers.tpl @@ -63,20 +63,6 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} {{- printf "%s:%s" $imageRepository $imageTag }} {{- end -}} -{{/* -FeatureGates -*/}} -{{- define "kuberay.featureGates" -}} -{{- $features := "" }} -{{- range .Values.featureGates }} - {{- $str := printf "%s=%t," .name .enabled }} - {{- $features = print $features $str }} -{{- end }} -{{- with .Values.featureGates }} ---feature-gates={{ $features | trimSuffix "," }} -{{- end }} -{{- end }} - {{- /* Create the name of the service to use. */ -}} {{- define "kuberay-operator.service.name" -}} {{- include "kuberay-operator.fullname" . }} diff --git a/helm-chart/kuberay-operator/templates/deployment.yaml b/helm-chart/kuberay-operator/templates/deployment.yaml index 2df45291e53..abf853dfe07 100644 --- a/helm-chart/kuberay-operator/templates/deployment.yaml +++ b/helm-chart/kuberay-operator/templates/deployment.yaml @@ -39,48 +39,40 @@ spec: command: - {{ .Values.operatorCommand }} args: - {{- $argList := list -}} - {{- $argList = append $argList (include "kuberay.featureGates" . | trim) -}} - {{- if .Values.batchScheduler -}} - {{- if .Values.batchScheduler.enabled -}} - {{- $argList = append $argList "--enable-batch-scheduler" -}} - {{- end -}} - {{- if .Values.batchScheduler.name -}} - {{- $argList = append $argList (printf "--batch-scheduler=%s" .Values.batchScheduler.name) -}} - {{- end -}} - {{- end -}} - {{- $watchNamespace := "" -}} - {{- if and .Values.singleNamespaceInstall (not .Values.watchNamespace) -}} - {{- $watchNamespace = .Release.Namespace -}} - {{- else if .Values.watchNamespace -}} - {{- $watchNamespace = join "," .Values.watchNamespace -}} - {{- end -}} - {{- if $watchNamespace -}} - {{- $argList = append $argList "--watch-namespace" -}} - {{- $argList = append $argList $watchNamespace -}} - {{- end -}} - {{- if and (.Values.logging.baseDir) (.Values.logging.fileName) -}} - {{- $argList = append $argList "--log-file-path" -}} - {{- $argList = append $argList (printf "%s/%s" .Values.logging.baseDir .Values.logging.fileName) -}} - {{- end -}} - {{- if .Values.logging.stdoutEncoder -}} - {{- $argList = append $argList "--log-stdout-encoder" -}} - {{- $argList = append $argList .Values.logging.stdoutEncoder -}} - {{- end -}} - {{- if .Values.logging.fileEncoder -}} - {{- $argList = append $argList "--log-file-encoder" -}} - {{- $argList = append $argList .Values.logging.fileEncoder -}} - {{- end -}} - {{- if hasKey .Values "useKubernetesProxy" -}} - {{- $argList = append $argList (printf "--use-kubernetes-proxy=%t" .Values.useKubernetesProxy) -}} - {{- end -}} - {{- if hasKey .Values "leaderElectionEnabled" -}} - {{- $argList = append $argList (printf "--enable-leader-election=%t" .Values.leaderElectionEnabled) -}} - {{- end -}} - {{- if and (hasKey .Values "metrics") (hasKey .Values.metrics "enabled") }} - {{- $argList = append $argList (printf "--enable-metrics=%t" .Values.metrics.enabled) -}} - {{- end -}} - {{- $argList | toYaml | nindent 8 }} + {{- $featureGates := list }} + {{- range $featureGate := .Values.featureGates }} + {{- $featureGates = append $featureGates (printf "%s=%t" $featureGate.name $featureGate.enabled) }} + {{- end }} + {{- with $featureGates }} + - --feature-gates={{ . | join "," }} + {{- end }} + {{- if .Values.batchScheduler.enabled }} + - --enable-batch-scheduler + {{- end }} + {{- with .Values.batchScheduler.name }} + - --batch-scheduler={{ . }} + {{- end }} + {{- $namespaces := list }} + {{- if and .Values.singleNamespaceInstall (not .Values.watchNamespace) }} + {{- $namespaces = append $namespaces .Release.Namespace }} + {{- else if .Values.watchNamespace }} + {{- $namespaces = concat $namespaces .Values.watchNamespace }} + {{- end }} + {{- with $namespaces }} + - --watch-namespace={{ . | join "," }} + {{- end }} + {{- if and .Values.logging.baseDir .Values.logging.fileName }} + - --log-file-path={{ printf "%s/%s" .Values.logging.baseDir .Values.logging.fileName }} + {{- end }} + {{- with .Values.logging.stdoutEncoder }} + - --log-stdout-encoder={{ . }} + {{- end }} + {{- with .Values.logging.fileEncoder }} + - --log-file-encoder={{ . }} + {{- end }} + - --use-kubernetes-proxy={{ .Values.useKubernetesProxy }} + - --enable-leader-election={{ .Values.leaderElectionEnabled }} + - --enable-metrics={{ .Values.metrics.enabled }} {{- with .Values.env }} env: {{- toYaml . | nindent 8 }} diff --git a/helm-chart/kuberay-operator/values.yaml b/helm-chart/kuberay-operator/values.yaml index 0a9594007cf..03dd19fabac 100644 --- a/helm-chart/kuberay-operator/values.yaml +++ b/helm-chart/kuberay-operator/values.yaml @@ -101,9 +101,9 @@ metrics: # -- Path to the operator binary operatorCommand: /manager -# if userKubernetesProxy is set to true, the KubeRay operator will be configured with the --use-kubernetes-proxy flag. -# Using this option to configure kuberay-operator to comunitcate to Ray head pods by proxying through the Kubernetes API Server. -# useKubernetesProxy: true +# if useKubernetesProxy is set to true, the KubeRay operator will be configured with the --use-kubernetes-proxy flag. +# Using this option to configure kuberay-operator to communicate to Ray head pods by proxying through the Kubernetes API Server. +useKubernetesProxy: false # -- If leaderElectionEnabled is set to true, the KubeRay operator will use leader election for high availability. leaderElectionEnabled: true @@ -128,10 +128,10 @@ crNamespacedRbacEnable: true # to resource events within its own namespace. singleNamespaceInstall: false -# The KubeRay operator will watch the custom resources in the namespaces listed in the "watchNamespace" parameter. -# watchNamespace: -# - n1 -# - n2 +# -- The KubeRay operator will watch the custom resources in the namespaces listed in the "watchNamespace" parameter. +watchNamespace: +# - ns1 +# - ns2 # -- Environment variables. From fcb255c3fc98e02827c5086c3e6ca68c2f6bc9b3 Mon Sep 17 00:00:00 2001 From: Yi Chen Date: Thu, 29 May 2025 21:12:16 +0800 Subject: [PATCH 6/8] Define helper template for kuberay operator labels Signed-off-by: Yi Chen --- .../kuberay-operator/templates/_helpers.tpl | 21 ++++++++++++------- .../templates/deployment.yaml | 13 ++---------- .../kuberay-operator/templates/service.yaml | 3 +-- 3 files changed, 17 insertions(+), 20 deletions(-) diff --git a/helm-chart/kuberay-operator/templates/_helpers.tpl b/helm-chart/kuberay-operator/templates/_helpers.tpl index 4026dc8550f..f0d6d5d2c55 100644 --- a/helm-chart/kuberay-operator/templates/_helpers.tpl +++ b/helm-chart/kuberay-operator/templates/_helpers.tpl @@ -38,17 +38,24 @@ Create chart name and version as used by the chart label. {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} {{- end -}} -{{/* -Common labels -*/}} +{{- /* Common labels */ -}} {{- define "kuberay-operator.labels" -}} -app.kubernetes.io/name: {{ include "kuberay-operator.name" . }} helm.sh/chart: {{ include "kuberay-operator.chart" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{ include "kuberay-operator.selectorLabels" . }} +app.kubernetes.io/component: {{ include "kuberay-operator.component" . }} +{{- with .Chart.AppVersion }} +app.kubernetes.io/version: {{ . | quote }} {{- end }} app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- with .Values.labels }} +{{ . | toYaml }} +{{- end }} +{{- end -}} + +{{- /* Selector labels */ -}} +{{- define "kuberay-operator.selectorLabels" -}} +app.kubernetes.io/name: {{ include "kuberay-operator.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} {{- end -}} {{- /* Create the name of the deployment to use. */ -}} diff --git a/helm-chart/kuberay-operator/templates/deployment.yaml b/helm-chart/kuberay-operator/templates/deployment.yaml index abf853dfe07..70cc42e3acc 100644 --- a/helm-chart/kuberay-operator/templates/deployment.yaml +++ b/helm-chart/kuberay-operator/templates/deployment.yaml @@ -5,26 +5,17 @@ metadata: namespace: {{ .Release.Namespace }} labels: {{- include "kuberay-operator.labels" . | nindent 4 }} - {{- with .Values.labels }} - {{- toYaml . | nindent 4 }} - {{- end }} spec: replicas: 1 strategy: type: Recreate selector: matchLabels: - app.kubernetes.io/name: {{ include "kuberay-operator.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} + {{- include "kuberay-operator.selectorLabels" . | nindent 6 }} template: metadata: labels: - app.kubernetes.io/name: {{ include "kuberay-operator.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} - app.kubernetes.io/component: {{ include "kuberay-operator.component" . }} - {{- with .Values.labels }} - {{- toYaml . | nindent 8 }} - {{- end }} + {{- include "kuberay-operator.labels" . | nindent 8 }} {{- with .Values.annotations }} annotations: {{- toYaml . | nindent 8 }} diff --git a/helm-chart/kuberay-operator/templates/service.yaml b/helm-chart/kuberay-operator/templates/service.yaml index d240602f600..4a84929caa2 100644 --- a/helm-chart/kuberay-operator/templates/service.yaml +++ b/helm-chart/kuberay-operator/templates/service.yaml @@ -13,5 +13,4 @@ spec: protocol: TCP name: http selector: - app.kubernetes.io/name: {{ include "kuberay-operator.name" . }} - app.kubernetes.io/instance: {{ .Release.Name }} + {{- include "kuberay-operator.selectorLabels" . | nindent 4 }} From 734c4096fc4b53f585766399e28a60af924cc89b Mon Sep 17 00:00:00 2001 From: Yi Chen Date: Fri, 30 May 2025 11:48:11 +0800 Subject: [PATCH 7/8] Update kuberay operator Helm unit tests Signed-off-by: Yi Chen --- .../tests/deployment_test.yaml | 93 +++++++++++++++++++ helm-chart/kuberay-operator/values.yaml | 2 +- 2 files changed, 94 insertions(+), 1 deletion(-) diff --git a/helm-chart/kuberay-operator/tests/deployment_test.yaml b/helm-chart/kuberay-operator/tests/deployment_test.yaml index 2c11fe15e4d..9871e73e34a 100644 --- a/helm-chart/kuberay-operator/tests/deployment_test.yaml +++ b/helm-chart/kuberay-operator/tests/deployment_test.yaml @@ -139,6 +139,99 @@ tests: seccompProfile: type: RuntimeDefault + - it: Should add feature gates to arguments if specified + set: + featureGates: + - name: RayClusterStatusConditions + enabled: true + - name: RayJobDeletionPolicy + enabled: false + asserts: + - contains: + path: spec.template.spec.containers[?(@.name=="kuberay-operator")].args + content: --feature-gates=RayClusterStatusConditions=true,RayJobDeletionPolicy=false + + - it: Should enable batch scheduler if `batchScheduler.enabled` is set to `true` + set: + batchScheduler: + enabled: true + asserts: + - contains: + path: spec.template.spec.containers[?(@.name=="kuberay-operator")].args + content: --enable-batch-scheduler + + - it: Should use the specified batch scheduler name if `batchScheduler.name` is set + set: + batchScheduler: + name: test-batch-scheduler + asserts: + - contains: + path: spec.template.spec.containers[?(@.name=="kuberay-operator")].args + content: --batch-scheduler=test-batch-scheduler + + - it: Should watch the release namespace if `singleNamespaceInstall` is `true` and `watchNamespace` is empty + set: + singleNamespaceInstall: true + watchNamespace: [] + asserts: + - contains: + path: spec.template.spec.containers[?(@.name=="kuberay-operator")].args + content: --watch-namespace=kuberay-system + + - it: Should watch the specified namespaces if `watchNamespace` is set + set: + watchNamespace: + - ns1 + - ns2 + - ns3 + asserts: + - contains: + path: spec.template.spec.containers[?(@.name=="kuberay-operator")].args + content: --watch-namespace=ns1,ns2,ns3 + + - it: Should use the specified logging stdout encoder if `stdoutEncoder` is set + set: + logging: + stdoutEncoder: test-stdout-encoder + asserts: + - contains: + path: spec.template.spec.containers[?(@.name=="kuberay-operator")].args + content: --log-stdout-encoder=test-stdout-encoder + + - it: Should use the specified logging file encoder if `stdoutEncoder` is set + set: + logging: + fileEncoder: test-file-encoder + asserts: + - contains: + path: spec.template.spec.containers[?(@.name=="kuberay-operator")].args + content: --log-file-encoder=test-file-encoder + + - it: Should use kubernetes proxy if `useKubernetesProxy` is set to `true` + set: + useKubernetesProxy: true + asserts: + - contains: + path: spec.template.spec.containers[?(@.name=="kuberay-operator")].args + content: --use-kubernetes-proxy=true + + - it: Should enable leader election if `leaderElectionEnabled` is set to `true` + set: + leaderElectionEnabled: true + asserts: + - contains: + path: spec.template.spec.containers[?(@.name=="kuberay-operator")].args + content: --enable-leader-election=true + + - it: Should enable metrics if `metrics.enabled` is set to `true` + set: + metrics: + enabled: true + asserts: + - contains: + path: spec.template.spec.containers[?(@.name=="kuberay-operator")].args + content: --enable-metrics=true + - it: Should add environment variables if `env` is set set: env: diff --git a/helm-chart/kuberay-operator/values.yaml b/helm-chart/kuberay-operator/values.yaml index 03dd19fabac..c79d57b8429 100644 --- a/helm-chart/kuberay-operator/values.yaml +++ b/helm-chart/kuberay-operator/values.yaml @@ -129,7 +129,7 @@ crNamespacedRbacEnable: true singleNamespaceInstall: false # -- The KubeRay operator will watch the custom resources in the namespaces listed in the "watchNamespace" parameter. -watchNamespace: +watchNamespace: [] # - ns1 # - ns2 From 51568b5b9a7d46c784d3f2159953002daa093cad Mon Sep 17 00:00:00 2001 From: Yi Chen Date: Fri, 30 May 2025 11:48:58 +0800 Subject: [PATCH 8/8] Run make helm-docs Signed-off-by: Yi Chen --- helm-chart/kuberay-operator/README.md | 8 ++------ helm-chart/kuberay-operator/values.yaml | 4 ++-- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/helm-chart/kuberay-operator/README.md b/helm-chart/kuberay-operator/README.md index d5de18aa203..e7bfac36e71 100644 --- a/helm-chart/kuberay-operator/README.md +++ b/helm-chart/kuberay-operator/README.md @@ -172,18 +172,14 @@ spec: | metrics.serviceMonitor.selector | object | `{}` | Prometheus ServiceMonitor selector | | metrics.serviceMonitor.namespace | string | `""` | Prometheus ServiceMonitor namespace | | operatorCommand | string | `"/manager"` | Path to the operator binary | +| useKubernetesProxy | bool | `false` | | | leaderElectionEnabled | bool | `true` | If leaderElectionEnabled is set to true, the KubeRay operator will use leader election for high availability. | | rbacEnable | bool | `true` | If rbacEnable is set to false, no RBAC resources will be created, including the Role for leader election, the Role for Pods and Services, and so on. | | crNamespacedRbacEnable | bool | `true` | When crNamespacedRbacEnable is set to true, the KubeRay operator will create a Role for RayCluster preparation (e.g., Pods, Services) and a corresponding RoleBinding for each namespace listed in the "watchNamespace" parameter. Please note that even if crNamespacedRbacEnable is set to false, the Role and RoleBinding for leader election will still be created. Note: (1) This variable is only effective when rbacEnable and singleNamespaceInstall are both set to true. (2) In most cases, it should be set to true, unless you are using a Kubernetes cluster managed by GitOps tools such as ArgoCD. | | singleNamespaceInstall | bool | `false` | When singleNamespaceInstall is true: - Install namespaced RBAC resources such as Role and RoleBinding instead of cluster-scoped ones like ClusterRole and ClusterRoleBinding so that the chart can be installed by users with permissions restricted to a single namespace. (Please note that this excludes the CRDs, which can only be installed at the cluster scope.) - If "watchNamespace" is not set, the KubeRay operator will, by default, only listen to resource events within its own namespace. | +| watchNamespace | list | `[]` | The KubeRay operator will watch the custom resources in the namespaces listed in the "watchNamespace" parameter. | | env | string | `nil` | Environment variables. | | resources | object | `{"limits":{"cpu":"100m","memory":"512Mi"}}` | Resource requests and limits for containers. | -| livenessProbe.initialDelaySeconds | int | `10` | | -| livenessProbe.periodSeconds | int | `5` | | -| livenessProbe.failureThreshold | int | `5` | | -| readinessProbe.initialDelaySeconds | int | `10` | | -| readinessProbe.periodSeconds | int | `5` | | -| readinessProbe.failureThreshold | int | `5` | | | podSecurityContext | object | `{}` | Set up `securityContext` to improve Pod security. | | service.type | string | `"ClusterIP"` | Service type. | | service.port | int | `8080` | Service port. | diff --git a/helm-chart/kuberay-operator/values.yaml b/helm-chart/kuberay-operator/values.yaml index c79d57b8429..600b4c1e2b2 100644 --- a/helm-chart/kuberay-operator/values.yaml +++ b/helm-chart/kuberay-operator/values.yaml @@ -184,13 +184,13 @@ resources: # cpu: 100m # memory: 512Mi -# @Ignore -- Pod liveness probe configuration. +# @ignore -- Pod liveness probe configuration. livenessProbe: initialDelaySeconds: 10 periodSeconds: 5 failureThreshold: 5 -# @Ignore -- Pod readiness probe configuration. +# @ignore -- Pod readiness probe configuration. readinessProbe: initialDelaySeconds: 10 periodSeconds: 5