From 63ab287e58d8e1adf33e6e22e6aa6a4c7622fe2e Mon Sep 17 00:00:00 2001 From: Joe Lanford Date: Thu, 18 Sep 2025 17:35:21 -0400 Subject: [PATCH] Secure metrics endpoint with cntrlr-runtime metrics authz mechanics Signed-off-by: Anik Bhattacharjee --- .github/workflows/e2e-tests.yml | 4 +- Makefile | 53 +++++++- cmd/catalog/main.go | 12 +- cmd/olm/main.go | 19 ++- .../0000_50_olm_04-cert-manager.yaml | 46 +++++++ ...000_50_olm_07-olm-operator.deployment.yaml | 28 ++-- ...50_olm_08-catalog-operator.deployment.yaml | 26 ++-- deploy/chart/templates/_helpers.tpl | 2 +- deploy/chart/values.yaml | 19 ++- pkg/lib/server/server.go | 52 +++++++- vendor/modules.txt | 1 + .../pkg/metrics/filters/filters.go | 122 ++++++++++++++++++ 12 files changed, 329 insertions(+), 55 deletions(-) create mode 100644 deploy/chart/templates/0000_50_olm_04-cert-manager.yaml create mode 100644 vendor/sigs.k8s.io/controller-runtime/pkg/metrics/filters/filters.go diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index b806578931..be98ac9a06 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -86,7 +86,7 @@ jobs: for i in $(seq 1 ${E2E_NODES}); do KIND_CLUSTER_NAME="kind-olmv0-${i}" \ KIND_CREATE_OPTS="--kubeconfig=${E2E_KUBECONFIG_ROOT}/kubeconfig-${i}" \ - HELM_INSTALL_OPTS="--kubeconfig ${E2E_KUBECONFIG_ROOT}/kubeconfig-${i}" \ + HELM_INSTALL_OPTS="--kubeconfig ${E2E_KUBECONFIG_ROOT}/kubeconfig-${i} --set certManager.enabled=false" \ make kind-create deploy; done @@ -173,7 +173,7 @@ jobs: for i in $(seq 1 ${E2E_NODES}); do KIND_CLUSTER_NAME="kind-olmv0-${i}" \ KIND_CREATE_OPTS="--kubeconfig=${E2E_KUBECONFIG_ROOT}/kubeconfig-${i}" \ - HELM_INSTALL_OPTS="--kubeconfig ${E2E_KUBECONFIG_ROOT}/kubeconfig-${i}" \ + HELM_INSTALL_OPTS="--kubeconfig ${E2E_KUBECONFIG_ROOT}/kubeconfig-${i} --set certManager.enabled=false" \ make kind-create deploy; done diff --git a/Makefile b/Makefile index d10ff7ab85..99d636bb78 100644 --- a/Makefile +++ b/Makefile @@ -48,6 +48,12 @@ GINKGO := $(TOOL_EXEC) github.com/onsi/ginkgo/v2/ginkgo # Target environment and Dependencies # +# Cert-manager version - update this for new releases +CERT_MANAGER_VERSION ?= v1.18.2 + +# Cert-manager deployment timeout +CERT_MANAGER_TIMEOUT ?= 120s + # Minor Kubernetes version to build against derived from the client-go dependency version KUBE_MINOR ?= $(shell go list -m k8s.io/client-go | cut -d" " -f2 | sed 's/^v0\.\([[:digit:]]\{1,\}\)\.[[:digit:]]\{1,\}$$/1.\1/') @@ -157,7 +163,29 @@ local-build: IMAGE_TAG = local local-build: image .PHONY: run-local -run-local: local-build kind-create deploy +run-local: local-build kind-create cert-manager-install deploy + +.PHONY: cert-manager-install +cert-manager-install: #HELP Install cert-manager $(CERT_MANAGER_VERSION) + @echo "Installing cert-manager $(CERT_MANAGER_VERSION)" + kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/$(CERT_MANAGER_VERSION)/cert-manager.yaml + @echo "Waiting for cert-manager to be ready..." + kubectl wait --for=condition=Available --namespace=cert-manager deployment/cert-manager --timeout=$(CERT_MANAGER_TIMEOUT) + kubectl wait --for=condition=Available --namespace=cert-manager deployment/cert-manager-cainjector --timeout=$(CERT_MANAGER_TIMEOUT) + kubectl wait --for=condition=Available --namespace=cert-manager deployment/cert-manager-webhook --timeout=$(CERT_MANAGER_TIMEOUT) + @echo "Waiting for cert-manager webhook to be ready..." + kubectl wait --for=condition=Ready --namespace=cert-manager pod -l app=webhook --timeout=$(CERT_MANAGER_TIMEOUT) + @echo "Waiting for cert-manager CRDs to be available..." + kubectl wait --for condition=established --timeout=$(CERT_MANAGER_TIMEOUT) crd/certificates.cert-manager.io + kubectl wait --for condition=established --timeout=$(CERT_MANAGER_TIMEOUT) crd/issuers.cert-manager.io + @echo "cert-manager $(CERT_MANAGER_VERSION) installed successfully" + +.PHONY: cert-manager-uninstall +cert-manager-uninstall: #HELP Uninstall cert-manager + @echo "Uninstalling cert-manager..." + kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/$(CERT_MANAGER_VERSION)/cert-manager.yaml --ignore-not-found=true + @echo "cert-manager uninstalled" + .PHONY: clean clean: #HELP Clean up build artifacts @@ -231,6 +259,7 @@ deploy: $(KIND) $(HELM) #HELP Deploy OLM to kind cluster $KIND_CLUSTER_NAME (def $(KIND) load docker-image $(OLM_IMAGE) --name $(KIND_CLUSTER_NAME); \ $(HELM) upgrade --install olm deploy/chart \ --set debug=true \ + --set certManager.enabled=true \ --set olm.image.ref=$(OLM_IMAGE) \ --set olm.image.pullPolicy=IfNotPresent \ --set catalog.image.ref=$(OLM_IMAGE) \ @@ -254,6 +283,9 @@ undeploy: $(KIND) $(HELM) #HELP Uninstall OLM from kind cluster $KIND_CLUSTER_NA $(HELM) uninstall olm kubectl delete -f deploy/chart/crds + # Uninstall cert-manager + $(MAKE) cert-manager-uninstall + #SECTION e2e # E2E test configuration @@ -269,7 +301,24 @@ e2e: #HELP Run e2e tests against a cluster running OLM (params: $E2E_TEST_NS (op $(GO_TEST_ENV) $(GINKGO) -timeout $(E2E_TIMEOUT) $(GINKGO_OPTS) $(E2E_GINKGO_OPTS) ./test/e2e -- -namespace=$(E2E_TEST_NS) -olmNamespace=$(E2E_INSTALL_NS) -catalogNamespace=$(E2E_CATALOG_NS) $(E2E_OPTS) .PHONY: e2e-local -e2e-local: e2e-build kind-create deploy e2e +e2e-local: e2e-build kind-create e2e-local-deploy e2e + +.PHONY: e2e-local-deploy +e2e-local-deploy: $(KIND) $(HELM) #HELP Deploy OLM for e2e testing (without cert-manager) + $(KIND) load docker-image $(OLM_IMAGE) --name $(KIND_CLUSTER_NAME); \ + $(HELM) upgrade --install olm deploy/chart \ + --set debug=true \ + --set certManager.enabled=false \ + --set olm.image.ref=$(OLM_IMAGE) \ + --set olm.image.pullPolicy=IfNotPresent \ + --set catalog.image.ref=$(OLM_IMAGE) \ + --set catalog.image.pullPolicy=IfNotPresent \ + --set catalog.commandArgs=--configmapServerImage=$(CONFIGMAP_SERVER_IMAGE) \ + --set catalog.opmImageArgs=--opmImage=$(OPERATOR_REGISTRY_IMAGE) \ + --set package.image.ref=$(OLM_IMAGE) \ + --set package.image.pullPolicy=IfNotPresent \ + $(HELM_INSTALL_OPTS) \ + --wait; #SECTION Code Generation diff --git a/cmd/catalog/main.go b/cmd/catalog/main.go index b82f1689cb..4092bf8922 100644 --- a/cmd/catalog/main.go +++ b/cmd/catalog/main.go @@ -57,9 +57,16 @@ func (o *options) run(ctx context.Context, logger *logrus.Logger) error { o.catalogNamespace = catalogNamespaceEnvVarValue } + // create a config client for operator status + config, err := clientcmd.BuildConfigFromFlags("", o.kubeconfig) + if err != nil { + return fmt.Errorf("error configuring client: %s", err.Error()) + } + listenAndServe, err := server.GetListenAndServeFunc( server.WithLogger(logger), server.WithTLS(&o.tlsCertPath, &o.tlsKeyPath, &o.clientCAPath), + server.WithKubeConfig(config), server.WithDebug(o.debug), ) if err != nil { @@ -72,11 +79,6 @@ func (o *options) run(ctx context.Context, logger *logrus.Logger) error { } }() - // create a config client for operator status - config, err := clientcmd.BuildConfigFromFlags("", o.kubeconfig) - if err != nil { - return fmt.Errorf("error configuring client: %s", err.Error()) - } configClient, err := configv1client.NewForConfig(config) if err != nil { return fmt.Errorf("error configuring client: %s", err.Error()) diff --git a/cmd/olm/main.go b/cmd/olm/main.go index 715ae9aea0..10dab00f8c 100644 --- a/cmd/olm/main.go +++ b/cmd/olm/main.go @@ -123,7 +123,18 @@ func main() { } logger.Infof("log level %s", logger.Level) - listenAndServe, err := server.GetListenAndServeFunc(server.WithLogger(logger), server.WithTLS(tlsCertPath, tlsKeyPath, clientCAPath), server.WithDebug(*debug)) + mgr, err := Manager(ctx, *debug) + if err != nil { + logger.WithError(err).Fatal("error configuring controller manager") + } + config := mgr.GetConfig() + + listenAndServe, err := server.GetListenAndServeFunc( + server.WithLogger(logger), + server.WithTLS(tlsCertPath, tlsKeyPath, clientCAPath), + server.WithKubeConfig(config), + server.WithDebug(*debug), + ) if err != nil { logger.Fatalf("Error setting up health/metric/pprof service: %v", err) } @@ -134,12 +145,6 @@ func main() { } }() - mgr, err := Manager(ctx, *debug) - if err != nil { - logger.WithError(err).Fatal("error configuring controller manager") - } - config := mgr.GetConfig() - // create a config that validates we're creating objects with labels validatingConfig := validatingroundtripper.Wrap(config, mgr.GetScheme()) diff --git a/deploy/chart/templates/0000_50_olm_04-cert-manager.yaml b/deploy/chart/templates/0000_50_olm_04-cert-manager.yaml new file mode 100644 index 0000000000..699e8deef3 --- /dev/null +++ b/deploy/chart/templates/0000_50_olm_04-cert-manager.yaml @@ -0,0 +1,46 @@ +{{- if .Values.certManager.enabled }} +--- +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ .Values.certManager.issuer.name }} + namespace: {{ .Values.namespace }} +spec: + {{- if .Values.certManager.issuer.selfSigned }} + selfSigned: {} + {{- else if .Values.certManager.issuer.ca }} + ca: + secretName: {{ .Values.certManager.issuer.ca.secretName }} + {{- end }} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ .Values.certManager.certificate.name }} + namespace: {{ .Values.namespace }} +spec: + secretName: {{ .Values.certManager.certificate.secretName }} + isCA: false + usages: + - server auth + - client auth + dnsNames: + - localhost + - catalog-operator.{{ .Values.namespace }}.svc + - catalog-operator.{{ .Values.namespace }}.svc.cluster.local + - olm-operator.{{ .Values.namespace }}.svc + - olm-operator.{{ .Values.namespace }}.svc.cluster.local + {{- range .Values.certManager.certificate.extraDnsNames }} + - {{ . }} + {{- end }} + ipAddresses: + - 127.0.0.1 + {{- range .Values.certManager.certificate.extraIpAddresses }} + - {{ . }} + {{- end }} + issuerRef: + name: {{ .Values.certManager.issuer.name }} + kind: Issuer + group: cert-manager.io +{{- end }} + diff --git a/deploy/chart/templates/0000_50_olm_07-olm-operator.deployment.yaml b/deploy/chart/templates/0000_50_olm_07-olm-operator.deployment.yaml index 139f295195..342369963c 100644 --- a/deploy/chart/templates/0000_50_olm_07-olm-operator.deployment.yaml +++ b/deploy/chart/templates/0000_50_olm_07-olm-operator.deployment.yaml @@ -22,16 +22,14 @@ spec: seccompProfile: type: RuntimeDefault serviceAccountName: olm-operator-serviceaccount - volumes: - {{- if .Values.olm.tlsSecret }} + volumes: + {{- if .Values.certManager.enabled }} - name: srv-cert secret: - secretName: {{ .Values.olm.tlsSecret }} - {{- end }} - {{- if .Values.olm.clientCASecret }} + secretName: {{ .Values.certManager.certificate.secretName }} - name: profile-collector-cert secret: - secretName: {{ .Values.olm.clientCASecret }} + secretName: {{ .Values.certManager.certificate.secretName }} {{- end }} - name: tmpfs emptyDir: {} @@ -43,12 +41,10 @@ spec: capabilities: drop: [ "ALL" ] volumeMounts: - {{- if .Values.olm.tlsSecret }} + {{- if .Values.certManager.enabled }} - name: srv-cert mountPath: "/srv-cert" readOnly: true - {{- end }} - {{- if .Values.olm.clientCASecret }} - name: profile-collector-cert mountPath: "/profile-collector-cert" readOnly: true @@ -78,31 +74,29 @@ spec: - --writePackageServerStatusName - {{ .Values.writePackageServerStatusName }} {{- end }} - {{- if .Values.olm.tlsSecret }} + {{- if .Values.certManager.enabled }} - --tls-cert - /srv-cert/tls.crt - --tls-key - /srv-cert/tls.key - {{- end }} - {{- if .Values.olm.clientCASecret }} - --client-ca - /profile-collector-cert/tls.crt {{- end }} image: {{ .Values.olm.image.ref }} imagePullPolicy: {{ .Values.olm.image.pullPolicy }} ports: - - containerPort: {{ .Values.olm.service.internalPort }} + - containerPort: {{ if .Values.certManager.enabled }}{{ .Values.olm.service.internalPortHttps }}{{ else }}{{ .Values.olm.service.internalPort }}{{ end }} name: metrics livenessProbe: httpGet: path: /healthz - port: {{ .Values.olm.service.internalPort }} - scheme: {{ if .Values.olm.tlsSecret }}HTTPS{{ else }}HTTP{{end}} + port: {{ if .Values.certManager.enabled }}{{ .Values.olm.service.internalPortHttps }}{{ else }}{{ .Values.olm.service.internalPort }}{{ end }} + scheme: {{ if .Values.certManager.enabled }}HTTPS{{ else }}HTTP{{ end }} readinessProbe: httpGet: path: /healthz - port: {{ .Values.olm.service.internalPort }} - scheme: {{ if .Values.olm.tlsSecret }}HTTPS{{ else }}HTTP{{end}} + port: {{ if .Values.certManager.enabled }}{{ .Values.olm.service.internalPortHttps }}{{ else }}{{ .Values.olm.service.internalPort }}{{ end }} + scheme: {{ if .Values.certManager.enabled }}HTTPS{{ else }}HTTP{{ end }} terminationMessagePolicy: FallbackToLogsOnError env: - name: OPERATOR_NAMESPACE diff --git a/deploy/chart/templates/0000_50_olm_08-catalog-operator.deployment.yaml b/deploy/chart/templates/0000_50_olm_08-catalog-operator.deployment.yaml index 7b27706a74..5395b1f45f 100644 --- a/deploy/chart/templates/0000_50_olm_08-catalog-operator.deployment.yaml +++ b/deploy/chart/templates/0000_50_olm_08-catalog-operator.deployment.yaml @@ -23,15 +23,13 @@ spec: type: RuntimeDefault serviceAccountName: olm-operator-serviceaccount volumes: - {{- if .Values.catalog.tlsSecret }} + {{- if .Values.certManager.enabled }} - name: srv-cert secret: - secretName: {{ .Values.catalog.tlsSecret }} - {{- end }} - {{- if .Values.catalog.clientCASecret }} + secretName: {{ .Values.certManager.certificate.secretName }} - name: profile-collector-cert secret: - secretName: {{ .Values.catalog.clientCASecret }} + secretName: {{ .Values.certManager.certificate.secretName }} {{- end }} - name: tmpfs emptyDir: {} @@ -43,12 +41,10 @@ spec: capabilities: drop: [ "ALL" ] volumeMounts: - {{- if .Values.catalog.tlsSecret }} + {{- if .Values.certManager.enabled }} - name: srv-cert mountPath: "/srv-cert" readOnly: true - {{- end }} - {{- if .Values.catalog.clientCASecret }} - name: profile-collector-cert mountPath: "/profile-collector-cert" readOnly: true @@ -75,13 +71,11 @@ spec: - --writeStatusName - {{ .Values.writeStatusNameCatalog }} {{- end }} - {{- if .Values.catalog.tlsSecret }} + {{- if .Values.certManager.enabled }} - --tls-cert - /srv-cert/tls.crt - --tls-key - /srv-cert/tls.key - {{- end }} - {{- if .Values.catalog.clientCASecret }} - --client-ca - /profile-collector-cert/tls.crt {{- end }} @@ -98,18 +92,18 @@ spec: {{- end }} imagePullPolicy: {{ .Values.catalog.image.pullPolicy }} ports: - - containerPort: {{ .Values.olm.service.internalPort }} + - containerPort: {{ if .Values.certManager.enabled }}{{ .Values.catalog.service.internalPortHttps }}{{ else }}{{ .Values.catalog.service.internalPort }}{{ end }} name: metrics livenessProbe: httpGet: path: /healthz - port: {{ .Values.catalog.service.internalPort }} - scheme: {{ if .Values.catalog.tlsSecret }}HTTPS{{ else }}HTTP{{end}} + port: {{ if .Values.certManager.enabled }}{{ .Values.catalog.service.internalPortHttps }}{{ else }}{{ .Values.catalog.service.internalPort }}{{ end }} + scheme: {{ if .Values.certManager.enabled }}HTTPS{{ else }}HTTP{{ end }} readinessProbe: httpGet: path: /healthz - port: {{ .Values.catalog.service.internalPort }} - scheme: {{ if .Values.catalog.tlsSecret }}HTTPS{{ else }}HTTP{{end}} + port: {{ if .Values.certManager.enabled }}{{ .Values.catalog.service.internalPortHttps }}{{ else }}{{ .Values.catalog.service.internalPort }}{{ end }} + scheme: {{ if .Values.certManager.enabled }}HTTPS{{ else }}HTTP{{ end }} terminationMessagePolicy: FallbackToLogsOnError {{- if .Values.catalog.resources }} resources: diff --git a/deploy/chart/templates/_helpers.tpl b/deploy/chart/templates/_helpers.tpl index 308975b6f9..f0d83d2edb 100644 --- a/deploy/chart/templates/_helpers.tpl +++ b/deploy/chart/templates/_helpers.tpl @@ -13,4 +13,4 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this {{- define "fullname" -}} {{- $name := default .Chart.Name .Values.nameOverride -}} {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} \ No newline at end of file +{{- end -}} diff --git a/deploy/chart/values.yaml b/deploy/chart/values.yaml index 4e4ee726b8..416b4e419a 100644 --- a/deploy/chart/values.yaml +++ b/deploy/chart/values.yaml @@ -28,9 +28,8 @@ olm: pullPolicy: Always service: internalPort: 8080 + internalPortHttps: 8443 externalPort: metrics - # tlsSecret: olm-operator-serving-cert - # clientCASecret: pprof-serving-cert nodeSelector: kubernetes.io/os: linux resources: @@ -48,9 +47,8 @@ catalog: pullPolicy: Always service: internalPort: 8080 + internalPortHttps: 8443 externalPort: metrics - # tlsSecret: catalog-operator-serving-cert - # clientCASecret: pprof-serving-cert nodeSelector: kubernetes.io/os: linux resources: @@ -78,6 +76,19 @@ monitoring: enabled: false namespace: monitoring +certManager: + enabled: true + issuer: + name: olm-ca-issuer + selfSigned: true + ca: + secretName: "" + certificate: + name: olm-cert + secretName: olm-cert + extraDnsNames: [] + extraIpAddresses: [] + networkPolicy: dns: ports: diff --git a/pkg/lib/server/server.go b/pkg/lib/server/server.go index 3d79a192e0..d2ce396517 100644 --- a/pkg/lib/server/server.go +++ b/pkg/lib/server/server.go @@ -6,11 +6,15 @@ import ( "fmt" "net/http" "path/filepath" + "time" "github.com/operator-framework/operator-lifecycle-manager/pkg/lib/filemonitor" "github.com/operator-framework/operator-lifecycle-manager/pkg/lib/profile" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/metrics/filters" ) // Option applies a configuration option to the given config. @@ -43,11 +47,18 @@ func WithDebug(debug bool) Option { } } +func WithKubeConfig(config *rest.Config) Option { + return func(sc *serverConfig) { + sc.kubeConfig = config + } +} + type serverConfig struct { logger *logrus.Logger tlsCertPath *string tlsKeyPath *string clientCAPath *string + kubeConfig *rest.Config debug bool } @@ -62,6 +73,7 @@ func defaultServerConfig() serverConfig { tlsCertPath: nil, tlsKeyPath: nil, clientCAPath: nil, + kubeConfig: nil, logger: nil, debug: false, } @@ -90,12 +102,49 @@ func (sc serverConfig) getListenAndServeFunc() (func() error, error) { } mux := http.NewServeMux() - mux.Handle("/metrics", promhttp.Handler()) mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) }) profile.RegisterHandlers(mux, profile.WithTLS(tlsEnabled || !sc.debug)) + // Set up authenticated metrics endpoint if kubeConfig is provided + if sc.kubeConfig != nil && tlsEnabled { + sc.logger.Info("Setting up authenticated metrics endpoint") + // Create authentication filter using controller-runtime + filter, err := filters.WithAuthenticationAndAuthorization(sc.kubeConfig, &http.Client{ + Timeout: 30 * time.Second, + }) + if err != nil { + return nil, fmt.Errorf("failed to create authentication filter: %w", err) + } + // Create authenticated metrics handler + logger := log.FromContext(context.Background()) + authenticatedMetricsHandler, err := filter(logger, promhttp.Handler()) + if err != nil { + return nil, fmt.Errorf("failed to wrap metrics handler with authentication: %w", err) + } + // Add request logging for debugging if debug mode is enabled + if sc.debug { + debugAuthHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + sc.logger.Infof("Metrics request from %s, Auth header present: %v, User-Agent: %s", + r.RemoteAddr, r.Header.Get("Authorization") != "", r.Header.Get("User-Agent")) + authenticatedMetricsHandler.ServeHTTP(w, r) + }) + mux.Handle("/metrics", debugAuthHandler) + } else { + mux.Handle("/metrics", authenticatedMetricsHandler) + } + sc.logger.Info("Metrics endpoint configured with authentication and authorization") + } else { + // Fallback to unprotected metrics (for development/testing) + mux.Handle("/metrics", promhttp.Handler()) + if sc.kubeConfig == nil { + sc.logger.Warn("No Kubernetes config provided - metrics endpoint will be unprotected") + } else if !tlsEnabled { + sc.logger.Warn("TLS not enabled - metrics endpoint will be unprotected") + } + } + s := http.Server{ Handler: mux, Addr: sc.getAddress(tlsEnabled), @@ -141,6 +190,7 @@ func (sc serverConfig) getListenAndServeFunc() (func() error, error) { ClientAuth: tls.VerifyClientCertIfGiven, }, nil }, + NextProtos: []string{"http/1.1"}, // Disable HTTP/2 for security } return func() error { return s.ListenAndServeTLS("", "") diff --git a/vendor/modules.txt b/vendor/modules.txt index b4eeeecbd0..9b0cd4528e 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1873,6 +1873,7 @@ sigs.k8s.io/controller-runtime/pkg/log/zap sigs.k8s.io/controller-runtime/pkg/manager sigs.k8s.io/controller-runtime/pkg/manager/signals sigs.k8s.io/controller-runtime/pkg/metrics +sigs.k8s.io/controller-runtime/pkg/metrics/filters sigs.k8s.io/controller-runtime/pkg/metrics/server sigs.k8s.io/controller-runtime/pkg/predicate sigs.k8s.io/controller-runtime/pkg/reconcile diff --git a/vendor/sigs.k8s.io/controller-runtime/pkg/metrics/filters/filters.go b/vendor/sigs.k8s.io/controller-runtime/pkg/metrics/filters/filters.go new file mode 100644 index 0000000000..1659502bcf --- /dev/null +++ b/vendor/sigs.k8s.io/controller-runtime/pkg/metrics/filters/filters.go @@ -0,0 +1,122 @@ +package filters + +import ( + "fmt" + "net/http" + "strings" + "time" + + "github.com/go-logr/logr" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apiserver/pkg/apis/apiserver" + "k8s.io/apiserver/pkg/authentication/authenticatorfactory" + "k8s.io/apiserver/pkg/authorization/authorizer" + "k8s.io/apiserver/pkg/authorization/authorizerfactory" + authenticationv1 "k8s.io/client-go/kubernetes/typed/authentication/v1" + authorizationv1 "k8s.io/client-go/kubernetes/typed/authorization/v1" + "k8s.io/client-go/rest" + + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" +) + +// WithAuthenticationAndAuthorization provides a metrics.Filter for authentication and authorization. +// Metrics will be authenticated (via TokenReviews) and authorized (via SubjectAccessReviews) with the +// kube-apiserver. +// For the authentication and authorization the controller needs a ClusterRole +// with the following rules: +// * apiGroups: authentication.k8s.io, resources: tokenreviews, verbs: create +// * apiGroups: authorization.k8s.io, resources: subjectaccessreviews, verbs: create +// +// To scrape metrics e.g. via Prometheus the client needs a ClusterRole +// with the following rule: +// * nonResourceURLs: "/metrics", verbs: get +// +// Note: Please note that configuring this metrics provider will introduce a dependency to "k8s.io/apiserver" +// to your go module. +func WithAuthenticationAndAuthorization(config *rest.Config, httpClient *http.Client) (metricsserver.Filter, error) { + authenticationV1Client, err := authenticationv1.NewForConfigAndClient(config, httpClient) + if err != nil { + return nil, err + } + authorizationV1Client, err := authorizationv1.NewForConfigAndClient(config, httpClient) + if err != nil { + return nil, err + } + + authenticatorConfig := authenticatorfactory.DelegatingAuthenticatorConfig{ + Anonymous: &apiserver.AnonymousAuthConfig{Enabled: false}, // Require authentication. + CacheTTL: 1 * time.Minute, + TokenAccessReviewClient: authenticationV1Client, + TokenAccessReviewTimeout: 10 * time.Second, + // wait.Backoff is copied from: https://github.com/kubernetes/apiserver/blob/v0.29.0/pkg/server/options/authentication.go#L43-L50 + // options.DefaultAuthWebhookRetryBackoff is not used to avoid a dependency on "k8s.io/apiserver/pkg/server/options". + WebhookRetryBackoff: &wait.Backoff{ + Duration: 500 * time.Millisecond, + Factor: 1.5, + Jitter: 0.2, + Steps: 5, + }, + } + delegatingAuthenticator, _, err := authenticatorConfig.New() + if err != nil { + return nil, fmt.Errorf("failed to create authenticator: %w", err) + } + + authorizerConfig := authorizerfactory.DelegatingAuthorizerConfig{ + SubjectAccessReviewClient: authorizationV1Client, + AllowCacheTTL: 5 * time.Minute, + DenyCacheTTL: 30 * time.Second, + // wait.Backoff is copied from: https://github.com/kubernetes/apiserver/blob/v0.29.0/pkg/server/options/authentication.go#L43-L50 + // options.DefaultAuthWebhookRetryBackoff is not used to avoid a dependency on "k8s.io/apiserver/pkg/server/options". + WebhookRetryBackoff: &wait.Backoff{ + Duration: 500 * time.Millisecond, + Factor: 1.5, + Jitter: 0.2, + Steps: 5, + }, + } + delegatingAuthorizer, err := authorizerConfig.New() + if err != nil { + return nil, fmt.Errorf("failed to create authorizer: %w", err) + } + + return func(log logr.Logger, handler http.Handler) (http.Handler, error) { + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + ctx := req.Context() + + res, ok, err := delegatingAuthenticator.AuthenticateRequest(req) + if err != nil { + log.Error(err, "Authentication failed") + http.Error(w, "Authentication failed", http.StatusInternalServerError) + return + } + if !ok { + log.V(4).Info("Authentication failed") + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + + attributes := authorizer.AttributesRecord{ + User: res.User, + Verb: strings.ToLower(req.Method), + Path: req.URL.Path, + } + + authorized, reason, err := delegatingAuthorizer.Authorize(ctx, attributes) + if err != nil { + msg := fmt.Sprintf("Authorization for user %s failed", attributes.User.GetName()) + log.Error(err, msg) + http.Error(w, msg, http.StatusInternalServerError) + return + } + if authorized != authorizer.DecisionAllow { + msg := fmt.Sprintf("Authorization denied for user %s", attributes.User.GetName()) + log.V(4).Info(fmt.Sprintf("%s: %s", msg, reason)) + http.Error(w, msg, http.StatusForbidden) + return + } + + handler.ServeHTTP(w, req) + }), nil + }, nil +}