Skip to content

Secure metrics endpoint with cntrlr-runtime metrics authz mechanics #3660

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Secure metrics endpoint with cntrlr-runtime metrics authz mechanics #3660

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/workflows/e2e-tests.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ jobs:
for i in $(seq 1 ${E2E_NODES}); do
KIND_CLUSTER_NAME="kind-olmv0-${i}" \
KIND_CREATE_OPTS="--kubeconfig=${E2E_KUBECONFIG_ROOT}/kubeconfig-${i}" \
HELM_INSTALL_OPTS="--kubeconfig ${E2E_KUBECONFIG_ROOT}/kubeconfig-${i}" \
HELM_INSTALL_OPTS="--kubeconfig ${E2E_KUBECONFIG_ROOT}/kubeconfig-${i} --set certManager.enabled=false" \
Copy link
Contributor Author

@anik120 anik120 Sep 24, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This presented itself as the easiest way to do all the tests we have for metrics (since these tests are about testing the metrics omitted, eg "creating a subscription emits these metrics", and not the security aspect of the endpoints).

make kind-create deploy;
done

Expand Down Expand Up @@ -173,7 +173,7 @@ jobs:
for i in $(seq 1 ${E2E_NODES}); do
KIND_CLUSTER_NAME="kind-olmv0-${i}" \
KIND_CREATE_OPTS="--kubeconfig=${E2E_KUBECONFIG_ROOT}/kubeconfig-${i}" \
HELM_INSTALL_OPTS="--kubeconfig ${E2E_KUBECONFIG_ROOT}/kubeconfig-${i}" \
HELM_INSTALL_OPTS="--kubeconfig ${E2E_KUBECONFIG_ROOT}/kubeconfig-${i} --set certManager.enabled=false" \
make kind-create deploy;
done

Expand Down
53 changes: 51 additions & 2 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,12 @@ GINKGO := $(TOOL_EXEC) github.com/onsi/ginkgo/v2/ginkgo

# Target environment and Dependencies #

# Cert-manager version - update this for new releases
CERT_MANAGER_VERSION ?= v1.18.2

# Cert-manager deployment timeout
CERT_MANAGER_TIMEOUT ?= 120s

# Minor Kubernetes version to build against derived from the client-go dependency version
KUBE_MINOR ?= $(shell go list -m k8s.io/client-go | cut -d" " -f2 | sed 's/^v0\.\([[:digit:]]\{1,\}\)\.[[:digit:]]\{1,\}$$/1.\1/')

Expand Down Expand Up @@ -157,7 +163,29 @@ local-build: IMAGE_TAG = local
local-build: image

.PHONY: run-local
run-local: local-build kind-create deploy
run-local: local-build kind-create cert-manager-install deploy

.PHONY: cert-manager-install
cert-manager-install: #HELP Install cert-manager $(CERT_MANAGER_VERSION)
@echo "Installing cert-manager $(CERT_MANAGER_VERSION)"
kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/$(CERT_MANAGER_VERSION)/cert-manager.yaml
@echo "Waiting for cert-manager to be ready..."
kubectl wait --for=condition=Available --namespace=cert-manager deployment/cert-manager --timeout=$(CERT_MANAGER_TIMEOUT)
kubectl wait --for=condition=Available --namespace=cert-manager deployment/cert-manager-cainjector --timeout=$(CERT_MANAGER_TIMEOUT)
kubectl wait --for=condition=Available --namespace=cert-manager deployment/cert-manager-webhook --timeout=$(CERT_MANAGER_TIMEOUT)
@echo "Waiting for cert-manager webhook to be ready..."
kubectl wait --for=condition=Ready --namespace=cert-manager pod -l app=webhook --timeout=$(CERT_MANAGER_TIMEOUT)
@echo "Waiting for cert-manager CRDs to be available..."
kubectl wait --for condition=established --timeout=$(CERT_MANAGER_TIMEOUT) crd/certificates.cert-manager.io
kubectl wait --for condition=established --timeout=$(CERT_MANAGER_TIMEOUT) crd/issuers.cert-manager.io
@echo "cert-manager $(CERT_MANAGER_VERSION) installed successfully"

.PHONY: cert-manager-uninstall
cert-manager-uninstall: #HELP Uninstall cert-manager
@echo "Uninstalling cert-manager..."
kubectl delete -f https://github.com/cert-manager/cert-manager/releases/download/$(CERT_MANAGER_VERSION)/cert-manager.yaml --ignore-not-found=true
@echo "cert-manager uninstalled"


.PHONY: clean
clean: #HELP Clean up build artifacts
Expand Down Expand Up @@ -231,6 +259,7 @@ deploy: $(KIND) $(HELM) #HELP Deploy OLM to kind cluster $KIND_CLUSTER_NAME (def
$(KIND) load docker-image $(OLM_IMAGE) --name $(KIND_CLUSTER_NAME); \
$(HELM) upgrade --install olm deploy/chart \
--set debug=true \
--set certManager.enabled=true \
--set olm.image.ref=$(OLM_IMAGE) \
--set olm.image.pullPolicy=IfNotPresent \
--set catalog.image.ref=$(OLM_IMAGE) \
Expand All @@ -254,6 +283,9 @@ undeploy: $(KIND) $(HELM) #HELP Uninstall OLM from kind cluster $KIND_CLUSTER_NA
$(HELM) uninstall olm
kubectl delete -f deploy/chart/crds

# Uninstall cert-manager
$(MAKE) cert-manager-uninstall

#SECTION e2e

# E2E test configuration
Expand All @@ -269,7 +301,24 @@ e2e: #HELP Run e2e tests against a cluster running OLM (params: $E2E_TEST_NS (op
$(GO_TEST_ENV) $(GINKGO) -timeout $(E2E_TIMEOUT) $(GINKGO_OPTS) $(E2E_GINKGO_OPTS) ./test/e2e -- -namespace=$(E2E_TEST_NS) -olmNamespace=$(E2E_INSTALL_NS) -catalogNamespace=$(E2E_CATALOG_NS) $(E2E_OPTS)

.PHONY: e2e-local
e2e-local: e2e-build kind-create deploy e2e
e2e-local: e2e-build kind-create e2e-local-deploy e2e

.PHONY: e2e-local-deploy
e2e-local-deploy: $(KIND) $(HELM) #HELP Deploy OLM for e2e testing (without cert-manager)
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Also had to make a new deploy target to deploy olm without cert-manager for e2e testing

$(KIND) load docker-image $(OLM_IMAGE) --name $(KIND_CLUSTER_NAME); \
$(HELM) upgrade --install olm deploy/chart \
--set debug=true \
--set certManager.enabled=false \
--set olm.image.ref=$(OLM_IMAGE) \
--set olm.image.pullPolicy=IfNotPresent \
--set catalog.image.ref=$(OLM_IMAGE) \
--set catalog.image.pullPolicy=IfNotPresent \
--set catalog.commandArgs=--configmapServerImage=$(CONFIGMAP_SERVER_IMAGE) \
--set catalog.opmImageArgs=--opmImage=$(OPERATOR_REGISTRY_IMAGE) \
--set package.image.ref=$(OLM_IMAGE) \
--set package.image.pullPolicy=IfNotPresent \
$(HELM_INSTALL_OPTS) \
--wait;

#SECTION Code Generation

Expand Down
12 changes: 7 additions & 5 deletions cmd/catalog/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,9 +57,16 @@ func (o *options) run(ctx context.Context, logger *logrus.Logger) error {
o.catalogNamespace = catalogNamespaceEnvVarValue
}

// create a config client for operator status
config, err := clientcmd.BuildConfigFromFlags("", o.kubeconfig)
if err != nil {
return fmt.Errorf("error configuring client: %s", err.Error())
}

listenAndServe, err := server.GetListenAndServeFunc(
server.WithLogger(logger),
server.WithTLS(&o.tlsCertPath, &o.tlsKeyPath, &o.clientCAPath),
server.WithKubeConfig(config),
server.WithDebug(o.debug),
)
if err != nil {
Expand All @@ -72,11 +79,6 @@ func (o *options) run(ctx context.Context, logger *logrus.Logger) error {
}
}()

// create a config client for operator status
config, err := clientcmd.BuildConfigFromFlags("", o.kubeconfig)
if err != nil {
return fmt.Errorf("error configuring client: %s", err.Error())
}
configClient, err := configv1client.NewForConfig(config)
if err != nil {
return fmt.Errorf("error configuring client: %s", err.Error())
Expand Down
19 changes: 12 additions & 7 deletions cmd/olm/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,7 +123,18 @@ func main() {
}
logger.Infof("log level %s", logger.Level)

listenAndServe, err := server.GetListenAndServeFunc(server.WithLogger(logger), server.WithTLS(tlsCertPath, tlsKeyPath, clientCAPath), server.WithDebug(*debug))
mgr, err := Manager(ctx, *debug)
if err != nil {
logger.WithError(err).Fatal("error configuring controller manager")
}
config := mgr.GetConfig()

listenAndServe, err := server.GetListenAndServeFunc(
server.WithLogger(logger),
server.WithTLS(tlsCertPath, tlsKeyPath, clientCAPath),
server.WithKubeConfig(config),
server.WithDebug(*debug),
)
if err != nil {
logger.Fatalf("Error setting up health/metric/pprof service: %v", err)
}
Expand All @@ -134,12 +145,6 @@ func main() {
}
}()

mgr, err := Manager(ctx, *debug)
if err != nil {
logger.WithError(err).Fatal("error configuring controller manager")
}
config := mgr.GetConfig()

// create a config that validates we're creating objects with labels
validatingConfig := validatingroundtripper.Wrap(config, mgr.GetScheme())

Expand Down
46 changes: 46 additions & 0 deletions deploy/chart/templates/0000_50_olm_04-cert-manager.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
{{- if .Values.certManager.enabled }}
---
apiVersion: cert-manager.io/v1
kind: Issuer
metadata:
name: {{ .Values.certManager.issuer.name }}
namespace: {{ .Values.namespace }}
spec:
{{- if .Values.certManager.issuer.selfSigned }}
selfSigned: {}
{{- else if .Values.certManager.issuer.ca }}
ca:
secretName: {{ .Values.certManager.issuer.ca.secretName }}
{{- end }}
---
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
name: {{ .Values.certManager.certificate.name }}
namespace: {{ .Values.namespace }}
spec:
secretName: {{ .Values.certManager.certificate.secretName }}
isCA: false
usages:
- server auth
- client auth
dnsNames:
- localhost
- catalog-operator.{{ .Values.namespace }}.svc
- catalog-operator.{{ .Values.namespace }}.svc.cluster.local
- olm-operator.{{ .Values.namespace }}.svc
- olm-operator.{{ .Values.namespace }}.svc.cluster.local
{{- range .Values.certManager.certificate.extraDnsNames }}
- {{ . }}
{{- end }}
ipAddresses:
- 127.0.0.1
{{- range .Values.certManager.certificate.extraIpAddresses }}
- {{ . }}
{{- end }}
issuerRef:
name: {{ .Values.certManager.issuer.name }}
kind: Issuer
group: cert-manager.io
{{- end }}

28 changes: 11 additions & 17 deletions deploy/chart/templates/0000_50_olm_07-olm-operator.deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,16 +22,14 @@ spec:
seccompProfile:
type: RuntimeDefault
serviceAccountName: olm-operator-serviceaccount
volumes:
{{- if .Values.olm.tlsSecret }}
volumes:
{{- if .Values.certManager.enabled }}
- name: srv-cert
secret:
secretName: {{ .Values.olm.tlsSecret }}
{{- end }}
{{- if .Values.olm.clientCASecret }}
secretName: {{ .Values.certManager.certificate.secretName }}
- name: profile-collector-cert
secret:
secretName: {{ .Values.olm.clientCASecret }}
secretName: {{ .Values.certManager.certificate.secretName }}
{{- end }}
- name: tmpfs
emptyDir: {}
Expand All @@ -43,12 +41,10 @@ spec:
capabilities:
drop: [ "ALL" ]
volumeMounts:
{{- if .Values.olm.tlsSecret }}
{{- if .Values.certManager.enabled }}
- name: srv-cert
mountPath: "/srv-cert"
readOnly: true
{{- end }}
{{- if .Values.olm.clientCASecret }}
- name: profile-collector-cert
mountPath: "/profile-collector-cert"
readOnly: true
Expand Down Expand Up @@ -78,31 +74,29 @@ spec:
- --writePackageServerStatusName
- {{ .Values.writePackageServerStatusName }}
{{- end }}
{{- if .Values.olm.tlsSecret }}
{{- if .Values.certManager.enabled }}
- --tls-cert
- /srv-cert/tls.crt
- --tls-key
- /srv-cert/tls.key
{{- end }}
{{- if .Values.olm.clientCASecret }}
- --client-ca
- /profile-collector-cert/tls.crt
{{- end }}
image: {{ .Values.olm.image.ref }}
imagePullPolicy: {{ .Values.olm.image.pullPolicy }}
ports:
- containerPort: {{ .Values.olm.service.internalPort }}
- containerPort: {{ if .Values.certManager.enabled }}{{ .Values.olm.service.internalPortHttps }}{{ else }}{{ .Values.olm.service.internalPort }}{{ end }}
name: metrics
livenessProbe:
httpGet:
path: /healthz
port: {{ .Values.olm.service.internalPort }}
scheme: {{ if .Values.olm.tlsSecret }}HTTPS{{ else }}HTTP{{end}}
port: {{ if .Values.certManager.enabled }}{{ .Values.olm.service.internalPortHttps }}{{ else }}{{ .Values.olm.service.internalPort }}{{ end }}
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Which means the templates had to be updated to configure different endpoints based on the presence of the cert-manager

scheme: {{ if .Values.certManager.enabled }}HTTPS{{ else }}HTTP{{ end }}
readinessProbe:
httpGet:
path: /healthz
port: {{ .Values.olm.service.internalPort }}
scheme: {{ if .Values.olm.tlsSecret }}HTTPS{{ else }}HTTP{{end}}
port: {{ if .Values.certManager.enabled }}{{ .Values.olm.service.internalPortHttps }}{{ else }}{{ .Values.olm.service.internalPort }}{{ end }}
scheme: {{ if .Values.certManager.enabled }}HTTPS{{ else }}HTTP{{ end }}
terminationMessagePolicy: FallbackToLogsOnError
env:
- name: OPERATOR_NAMESPACE
Expand Down
26 changes: 10 additions & 16 deletions deploy/chart/templates/0000_50_olm_08-catalog-operator.deployment.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,13 @@ spec:
type: RuntimeDefault
serviceAccountName: olm-operator-serviceaccount
volumes:
{{- if .Values.catalog.tlsSecret }}
{{- if .Values.certManager.enabled }}
- name: srv-cert
secret:
secretName: {{ .Values.catalog.tlsSecret }}
{{- end }}
{{- if .Values.catalog.clientCASecret }}
secretName: {{ .Values.certManager.certificate.secretName }}
- name: profile-collector-cert
secret:
secretName: {{ .Values.catalog.clientCASecret }}
secretName: {{ .Values.certManager.certificate.secretName }}
{{- end }}
- name: tmpfs
emptyDir: {}
Expand All @@ -43,12 +41,10 @@ spec:
capabilities:
drop: [ "ALL" ]
volumeMounts:
{{- if .Values.catalog.tlsSecret }}
{{- if .Values.certManager.enabled }}
- name: srv-cert
mountPath: "/srv-cert"
readOnly: true
{{- end }}
{{- if .Values.catalog.clientCASecret }}
- name: profile-collector-cert
mountPath: "/profile-collector-cert"
readOnly: true
Expand All @@ -75,13 +71,11 @@ spec:
- --writeStatusName
- {{ .Values.writeStatusNameCatalog }}
{{- end }}
{{- if .Values.catalog.tlsSecret }}
{{- if .Values.certManager.enabled }}
- --tls-cert
- /srv-cert/tls.crt
- --tls-key
- /srv-cert/tls.key
{{- end }}
{{- if .Values.catalog.clientCASecret }}
- --client-ca
- /profile-collector-cert/tls.crt
{{- end }}
Expand All @@ -98,18 +92,18 @@ spec:
{{- end }}
imagePullPolicy: {{ .Values.catalog.image.pullPolicy }}
ports:
- containerPort: {{ .Values.olm.service.internalPort }}
- containerPort: {{ if .Values.certManager.enabled }}{{ .Values.catalog.service.internalPortHttps }}{{ else }}{{ .Values.catalog.service.internalPort }}{{ end }}
name: metrics
livenessProbe:
httpGet:
path: /healthz
port: {{ .Values.catalog.service.internalPort }}
scheme: {{ if .Values.catalog.tlsSecret }}HTTPS{{ else }}HTTP{{end}}
port: {{ if .Values.certManager.enabled }}{{ .Values.catalog.service.internalPortHttps }}{{ else }}{{ .Values.catalog.service.internalPort }}{{ end }}
scheme: {{ if .Values.certManager.enabled }}HTTPS{{ else }}HTTP{{ end }}
readinessProbe:
httpGet:
path: /healthz
port: {{ .Values.catalog.service.internalPort }}
scheme: {{ if .Values.catalog.tlsSecret }}HTTPS{{ else }}HTTP{{end}}
port: {{ if .Values.certManager.enabled }}{{ .Values.catalog.service.internalPortHttps }}{{ else }}{{ .Values.catalog.service.internalPort }}{{ end }}
scheme: {{ if .Values.certManager.enabled }}HTTPS{{ else }}HTTP{{ end }}
terminationMessagePolicy: FallbackToLogsOnError
{{- if .Values.catalog.resources }}
resources:
Expand Down
2 changes: 1 addition & 1 deletion deploy/chart/templates/_helpers.tpl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,4 @@ We truncate at 63 chars because some Kubernetes name fields are limited to this
{{- define "fullname" -}}
{{- $name := default .Chart.Name .Values.nameOverride -}}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
{{- end -}}
{{- end -}}
Loading