Enable Webhook Warning messages, add Gateway API integration and simply Ingress integration

api/apps/v1alpha1/common_types.go

@@ -17,11 +17,14 @@ limitations under the License.
 package v1alpha1
 
 import (
+	"fmt"
+
 	promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 	autoscalingv2 "k8s.io/api/autoscaling/v2"
 	corev1 "k8s.io/api/core/v1"
 	networkingv1 "k8s.io/api/networking/v1"
 	"k8s.io/utils/ptr"
+	gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
 )
 
 const (
@@ -38,9 +41,58 @@ const (
 // Expose defines attributes to expose the service.
 type Expose struct {
 	Service Service `json:"service,omitempty"`
+	// Deprecated: Use .spec.router instead.
 	Ingress Ingress `json:"ingress,omitempty"`
 }
 
+// +kubebuilder:validation:XValidation:rule="!(has(self.gateway) && has(self.ingress))", message="ingress and gateway cannot be specified together"
+type Router struct {
+	// HostDomainName is the domain name of the hostname matched by the router.
+	// The hostname is constructed as "<nimServiceName>.<namespace>.<hostDomainName>", where the <nimServiceName> a subdomain of the matched hostname.
+	// eg. example.com for "<nimServiceName>.<namespace>.example.com"
+	// +kubebuilder:validation:MinLength=1
+	// +kubebuilder:validation:MaxLength=63
+	// +kubebuilder:validation:Pattern=`^(([a-z0-9][a-z0-9\-]*[a-z0-9])|[a-z0-9]+\.)*([a-z]+|xn\-\-[a-z0-9]+)\.?$`
+	HostDomainName string `json:"hostDomainName,omitempty"`
+
+	// Annotations for the router, e.g. for ingress class or gateway
+	Annotations map[string]string `json:"annotations,omitempty"`
+
+	// Ingress is the ingress controller to use for the created ingress.
+	Ingress *RouterIngress `json:"ingress,omitempty"`
+
+	// Gateway is the gateway to use for the created HTTPRoute.
+	Gateway *Gateway `json:"gateway,omitempty"`
+}
+
+type RouterIngress struct {
+	// +kubebuilder:validation:MinLength=1
+	// IngressClass is the ingress class to use for the created ingress.
+	IngressClass string `json:"ingressClass"`
+
+	// TLSSecretName is the name of the secret containing the TLS certificate and key.
+	TLSSecretName string `json:"tlsSecretName,omitempty"`
+}
+
+type Gateway struct {
+	// +kubebuilder:validation:MinLength=1
+	// Namespace of the gateway
+	Namespace string `json:"namespace"`
+	// +kubebuilder:validation:MinLength=1
+	// Name of the gateway
+	Name string `json:"name"`
+
+	// +kubebuilder:default:=true
+	// HTTPRoutesEnabled is a flag to enable HTTPRoutes for the created gateway.
+	HTTPRoutesEnabled bool `json:"httpRoutesEnabled,omitempty"`
+}
+
+// DEPRECATED ExposeV1 defines attributes to expose the service.
+type ExposeV1 struct {
+	Service Service   `json:"service,omitempty"`
+	Ingress IngressV1 `json:"ingress,omitempty"`
+}
+
 // Service defines attributes to create a service.
 type Service struct {
 	Type corev1.ServiceType `json:"type,omitempty"`
@@ -67,10 +119,13 @@ type Service struct {
 	Annotations map[string]string `json:"annotations,omitempty"`
 }
 
-// ExposeV1 defines attributes to expose the service.
-type ExposeV1 struct {
-	Service Service   `json:"service,omitempty"`
-	Ingress IngressV1 `json:"ingress,omitempty"`
+// Deprecated: Use .spec.router.ingress instead.
+// IngressV1 defines attributes for ingress
+// +kubebuilder:validation:XValidation:rule="(has(self.spec) && has(self.enabled) && self.enabled) || !has(self.enabled) || !self.enabled", message="spec cannot be nil when ingress is enabled"
+type IngressV1 struct {
+	Enabled     *bool             `json:"enabled,omitempty"`
+	Annotations map[string]string `json:"annotations,omitempty"`
+	Spec        *IngressSpec      `json:"spec,omitempty"`
 }
 
 // Metrics defines attributes to setup metrics collection.
@@ -119,15 +174,6 @@ type Ingress struct {
 	Spec        networkingv1.IngressSpec `json:"spec,omitempty"`
 }
 
-// IngressV1 defines attributes for ingress
-//
-// +kubebuilder:validation:XValidation:rule="(has(self.spec) && has(self.enabled) && self.enabled) || !has(self.enabled) || !self.enabled", message="spec cannot be nil when ingress is enabled"
-type IngressV1 struct {
-	Enabled     *bool             `json:"enabled,omitempty"`
-	Annotations map[string]string `json:"annotations,omitempty"`
-	Spec        *IngressSpec      `json:"spec,omitempty"`
-}
-
 // ResourceRequirements defines the resources required for a container.
 type ResourceRequirements struct {
 	// Limits describes the maximum amount of compute resources allowed.
@@ -184,6 +230,100 @@ func (i *IngressV1) GenerateNetworkingV1IngressSpec(name string) networkingv1.In
 	return ingressSpec
 }
 
+func (r *Router) GenerateGatewayHTTPRouteSpec(namespace, name string, port int32) gatewayv1.HTTPRouteSpec {
+	if r.Gateway == nil || !r.Gateway.HTTPRoutesEnabled {
+		return gatewayv1.HTTPRouteSpec{}
+	}
+
+	return gatewayv1.HTTPRouteSpec{
+		CommonRouteSpec: gatewayv1.CommonRouteSpec{
+			ParentRefs: []gatewayv1.ParentReference{
+				{
+					Name:      gatewayv1.ObjectName(r.Gateway.Name),
+					Namespace: ptr.To(gatewayv1.Namespace(r.Gateway.Namespace)),
+				},
+			},
+		},
+		Hostnames: []gatewayv1.Hostname{gatewayv1.Hostname(r.getHostname(namespace, name))},
+		Rules: []gatewayv1.HTTPRouteRule{
+			{
+				BackendRefs: []gatewayv1.HTTPBackendRef{
+					{
+						BackendRef: gatewayv1.BackendRef{
+							BackendObjectReference: gatewayv1.BackendObjectReference{
+								Name: gatewayv1.ObjectName(name),
+								Port: ptr.To(gatewayv1.PortNumber(port)),
+							},
+						},
+					},
+				},
+				Matches: []gatewayv1.HTTPRouteMatch{
+					{
+						Path: &gatewayv1.HTTPPathMatch{
+							Type:  ptr.To(gatewayv1.PathMatchPathPrefix),
+							Value: ptr.To("/"),
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
+func (r *Router) getHostname(namespace, name string) string {
+	return fmt.Sprintf("%s.%s.%s", name, namespace, r.HostDomainName)
+}
+
+func (r *Router) GenerateIngressSpec(namespace, name string) networkingv1.IngressSpec {
+	if r.Ingress == nil {
+		return networkingv1.IngressSpec{}
+	}
+
+	ingressSpec := networkingv1.IngressSpec{
+		IngressClassName: ptr.To(r.Ingress.IngressClass),
+		Rules: []networkingv1.IngressRule{
+			{
+				Host: r.getHostname(namespace, name),
+				IngressRuleValue: networkingv1.IngressRuleValue{
+					HTTP: &networkingv1.HTTPIngressRuleValue{
+						Paths: []networkingv1.HTTPIngressPath{
+							{
+								Path:     "/",
+								PathType: ptr.To(networkingv1.PathTypePrefix),
+								Backend: networkingv1.IngressBackend{
+									Service: &networkingv1.IngressServiceBackend{
+										Name: name,
+										Port: networkingv1.ServiceBackendPort{
+											Name: DefaultNamedPortAPI,
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+
+	if r.Ingress.TLSSecretName != "" {
+		ingressSpec.TLS = []networkingv1.IngressTLS{
+			{
+				Hosts:      []string{r.getHostname(namespace, name)},
+				SecretName: r.Ingress.TLSSecretName,
+			},
+		}
+	}
+	return ingressSpec
+}
+
+func (r *Expose) GenerateIngressSpec(name string) networkingv1.IngressSpec {
+	if r.Ingress.Enabled == nil || !*r.Ingress.Enabled {
+		return networkingv1.IngressSpec{}
+	}
+	return r.Ingress.Spec
+}
+
 type IngressSpec struct {
 	// +kubebuilder:validation:Pattern=`[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*`
 	IngressClassName string        `json:"ingressClassName"`

api/apps/v1alpha1/nemo_customizer_types.go

@@ -33,6 +33,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/utils/ptr"
+	gatewayv1 "sigs.k8s.io/gateway-api/apis/v1"
 
 	rendertypes "github.com/NVIDIA/k8s-nim-operator/internal/render/types"
 	utils "github.com/NVIDIA/k8s-nim-operator/internal/utils"
@@ -66,6 +67,7 @@ const (
 )
 
 // NemoCustomizerSpec defines the desired state of NemoCustomizer.
+// +kubebuilder:validation:XValidation:rule="!(has(self.expose.ingress) && has(self.expose.ingress.enabled) && self.expose.ingress.enabled && has(self.router) && has(self.router.ingress))", message=".spec.expose.ingress is deprecated, and will be removed in a future release. If .spec.expose.ingress is set, please do not set .spec.router.ingress."
 type NemoCustomizerSpec struct {
 	Image        Image                        `json:"image"`
 	Command      []string                     `json:"command,omitempty"`
@@ -80,6 +82,7 @@ type NemoCustomizerSpec struct {
 	// +kubebuilder:validation:XValidation:rule="!(has(self.service.grpcPort))", message="unsupported field: spec.expose.service.grpcPort"
 	// +kubebuilder:validation:XValidation:rule="!(has(self.service.metricsPort))", message="unsupported field: spec.expose.service.metricsPort"
 	Expose  ExposeV1    `json:"expose,omitempty"`
+	Router  Router      `json:"router,omitempty"`
 	Scale   Autoscaling `json:"scale,omitempty"`
 	Metrics Metrics     `json:"metrics,omitempty"`
 
@@ -635,12 +638,25 @@ func (n *NemoCustomizer) IsAutoScalingEnabled() bool {
 
 // IsIngressEnabled returns true if ingress is enabled for NemoCustomizer deployment.
 func (n *NemoCustomizer) IsIngressEnabled() bool {
-	return n.Spec.Expose.Ingress.Enabled != nil && *n.Spec.Expose.Ingress.Enabled
+	return (n.Spec.Router.Ingress != nil && n.Spec.Router.Ingress.IngressClass != "") ||
+		(n.Spec.Expose.Ingress.Enabled != nil && *n.Spec.Expose.Ingress.Enabled) // TODO deprecate this once we have removed the .spec.expose.ingress field from the spec
 }
 
 // GetIngressSpec returns the Ingress spec NemoCustomizer deployment.
 func (n *NemoCustomizer) GetIngressSpec() networkingv1.IngressSpec {
-	return n.Spec.Expose.Ingress.GenerateNetworkingV1IngressSpec(n.GetName())
+	// TODO deprecate this once we have removed the .spec.expose.ingress field from the spec
+	if n.Spec.Expose.Ingress.Enabled != nil && *n.Spec.Expose.Ingress.Enabled {
+		return n.Spec.Expose.Ingress.GenerateNetworkingV1IngressSpec(n.GetName())
+	}
+	return n.Spec.Router.GenerateIngressSpec(n.GetNamespace(), n.GetName())
+}
+
+func (n *NemoCustomizer) IsHTTPRouteEnabled() bool {
+	return n.Spec.Router.Gateway != nil && n.Spec.Router.Gateway.HTTPRoutesEnabled
+}
+
+func (n *NemoCustomizer) GetHTTPRouteSpec() gatewayv1.HTTPRouteSpec {
+	return n.Spec.Router.GenerateGatewayHTTPRouteSpec(n.GetNamespace(), n.GetName(), n.GetServicePort())
 }
 
 // IsServiceMonitorEnabled returns true if servicemonitor is enabled for NemoCustomizer deployment.
@@ -839,6 +855,20 @@ func (n *NemoCustomizer) GetIngressParams() *rendertypes.IngressParams {
 	return params
 }
 
+// GetHTTPRouteParams returns params to render HTTPRoute from templates.
+func (n *NemoCustomizer) GetHTTPRouteParams() *rendertypes.HTTPRouteParams {
+	params := &rendertypes.HTTPRouteParams{}
+	params.Enabled = n.IsHTTPRouteEnabled()
+
+	// Set metadata
+	params.Name = n.GetName()
+	params.Namespace = n.GetNamespace()
+	params.Labels = n.GetServiceLabels()
+	params.Annotations = n.GetHTTPRouteAnnotations()
+	params.Spec = n.GetHTTPRouteSpec()
+	return params
+}
+
 // GetRoleParams returns params to render Role from templates.
 func (n *NemoCustomizer) GetRoleParams() *rendertypes.RoleParams {
 	params := &rendertypes.RoleParams{}
@@ -1017,12 +1047,25 @@ func (n *NemoCustomizer) GetServicePort() int32 {
 func (n *NemoCustomizer) GetIngressAnnotations() map[string]string {
 	NemoCustomizerAnnotations := n.GetNemoCustomizerAnnotations()
 
-	if n.Spec.Expose.Ingress.Annotations != nil {
+	// TODO deprecate this once we have removed the .spec.expose.ingress field from the spec
+	if n.Spec.Expose.Ingress.Enabled != nil && *n.Spec.Expose.Ingress.Enabled {
 		return utils.MergeMaps(NemoCustomizerAnnotations, n.Spec.Expose.Ingress.Annotations)
 	}
+	if n.Spec.Router.Annotations != nil {
+		return utils.MergeMaps(NemoCustomizerAnnotations, n.Spec.Router.Annotations)
+	}
 	return NemoCustomizerAnnotations
 }
 
+func (n *NemoCustomizer) GetHTTPRouteAnnotations() map[string]string {
+	annotations := n.GetNemoCustomizerAnnotations()
+
+	if n.Spec.Router.Annotations != nil {
+		return utils.MergeMaps(annotations, n.Spec.Router.Annotations)
+	}
+	return annotations
+}
+
 // GetServiceAnnotations return standard and customized service annotations.
 func (n *NemoCustomizer) GetServiceAnnotations() map[string]string {
 	NemoCustomizerAnnotations := n.GetNemoCustomizerAnnotations()