From e4e234838cd74e09fbf1d0e0e6e06f2e0809c05c Mon Sep 17 00:00:00 2001 From: alekspog Date: Fri, 24 Mar 2023 13:54:42 +0200 Subject: [PATCH 1/9] add controller test Signed-off-by: alekspog --- internal/controllers/alert_controller_test.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/internal/controllers/alert_controller_test.go b/internal/controllers/alert_controller_test.go index 89e369758..e928eb0ea 100644 --- a/internal/controllers/alert_controller_test.go +++ b/internal/controllers/alert_controller_test.go @@ -267,6 +267,11 @@ func TestAlertReconciler_EventHandler(t *testing.T) { Name: "*", Namespace: "test", }, + { + Kind: "Kustomization", + Name: "testwildcardnamespace", + Namespace: "*", + }, }, ExclusionList: []string{ "doesnotoccur", // not intended to match @@ -375,6 +380,17 @@ func TestAlertReconciler_EventHandler(t *testing.T) { }, forwarded: true, }, + { + name: "forwards events when namespace wildcard is used", + modifyEventFunc: func(e eventv1.Event) eventv1.Event { + e.InvolvedObject.Kind = "Kustomization" + e.InvolvedObject.Name = "testwildcardnamespace" + e.InvolvedObject.Namespace = "test-" + randStringRunes(5) + e.Message = "test" + return e + }, + forwarded: true, + }, { name: "forwards events when the label matches", modifyEventFunc: func(e eventv1.Event) eventv1.Event { From 30f68a5c687af2f3a44110f6b1ba8480688aacd5 Mon Sep 17 00:00:00 2001 From: alekspog Date: Fri, 24 Mar 2023 14:05:29 +0200 Subject: [PATCH 2/9] Allow wildcard namespaces Signed-off-by: alekspog --- internal/server/event_handlers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/server/event_handlers.go b/internal/server/event_handlers.go index 51ae967e0..06cd86684 100644 --- a/internal/server/event_handlers.go +++ b/internal/server/event_handlers.go @@ -288,7 +288,7 @@ func (s *EventServer) handleEvent() func(w http.ResponseWriter, r *http.Request) } func (s *EventServer) eventMatchesAlert(ctx context.Context, event *eventv1.Event, source apiv1.CrossNamespaceObjectReference, severity string) bool { - if event.InvolvedObject.Namespace == source.Namespace && event.InvolvedObject.Kind == source.Kind { + if event.InvolvedObject.Namespace == source.Namespace && event.InvolvedObject.Kind == source.Kind || source.Namespace == "*" { if event.Severity == severity || severity == eventv1.EventSeverityInfo { labelMatch := true if source.Name == "*" && source.MatchLabels != nil { From 9f760304d835f0fef009ae1939b989817a989148 Mon Sep 17 00:00:00 2001 From: alekspog Date: Thu, 30 Mar 2023 21:18:34 +0300 Subject: [PATCH 3/9] add the namespace wildcard Signed-off-by: alekspog --- internal/server/event_handlers.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/server/event_handlers.go b/internal/server/event_handlers.go index 06cd86684..532cfc901 100644 --- a/internal/server/event_handlers.go +++ b/internal/server/event_handlers.go @@ -288,7 +288,7 @@ func (s *EventServer) handleEvent() func(w http.ResponseWriter, r *http.Request) } func (s *EventServer) eventMatchesAlert(ctx context.Context, event *eventv1.Event, source apiv1.CrossNamespaceObjectReference, severity string) bool { - if event.InvolvedObject.Namespace == source.Namespace && event.InvolvedObject.Kind == source.Kind || source.Namespace == "*" { + if (event.InvolvedObject.Namespace == source.Namespace || source.Namespace == "*") && event.InvolvedObject.Kind == source.Kind { if event.Severity == severity || severity == eventv1.EventSeverityInfo { labelMatch := true if source.Name == "*" && source.MatchLabels != nil { From c684faf319893a327b12727454bbe10b20c34ca5 Mon Sep 17 00:00:00 2001 From: alekspog Date: Thu, 30 Mar 2023 21:22:45 +0300 Subject: [PATCH 4/9] add tests for enabled crossnamespaceref Signed-off-by: alekspog --- internal/controllers/alert_controller_test.go | 290 +++++++++++++++++- 1 file changed, 274 insertions(+), 16 deletions(-) diff --git a/internal/controllers/alert_controller_test.go b/internal/controllers/alert_controller_test.go index e928eb0ea..0d651828b 100644 --- a/internal/controllers/alert_controller_test.go +++ b/internal/controllers/alert_controller_test.go @@ -267,11 +267,6 @@ func TestAlertReconciler_EventHandler(t *testing.T) { Name: "*", Namespace: "test", }, - { - Kind: "Kustomization", - Name: "testwildcardnamespace", - Namespace: "*", - }, }, ExclusionList: []string{ "doesnotoccur", // not intended to match @@ -380,17 +375,6 @@ func TestAlertReconciler_EventHandler(t *testing.T) { }, forwarded: true, }, - { - name: "forwards events when namespace wildcard is used", - modifyEventFunc: func(e eventv1.Event) eventv1.Event { - e.InvolvedObject.Kind = "Kustomization" - e.InvolvedObject.Name = "testwildcardnamespace" - e.InvolvedObject.Namespace = "test-" + randStringRunes(5) - e.Message = "test" - return e - }, - forwarded: true, - }, { name: "forwards events when the label matches", modifyEventFunc: func(e eventv1.Event) eventv1.Event { @@ -441,3 +425,277 @@ func TestAlertReconciler_EventHandler(t *testing.T) { }) } } + +func TestAlertReconciler_EventHandler_CrossNamespaceRefs(t *testing.T) { + g := NewWithT(t) + var ( + namespace = "events-" + randStringRunes(5) + req *http.Request + provider *apiv1.Provider + ) + g.Expect(createNamespace(namespace)).NotTo(HaveOccurred(), "failed to create test namespace") + + eventMdlw := middleware.New(middleware.Config{ + Recorder: prommetrics.NewRecorder(prommetrics.Config{ + Prefix: "gotk_event", + }), + }) + + store, err := memorystore.New(&memorystore.Config{ + Interval: 5 * time.Minute, + }) + if err != nil { + t.Fatalf("failed to create memory storage") + } + + eventServer := server.NewEventServer("127.0.0.1:56789", logf.Log, k8sClient, false) + stopCh := make(chan struct{}) + go eventServer.ListenAndServe(stopCh, eventMdlw, store) + + rcvServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + req = r + w.WriteHeader(200) + })) + defer rcvServer.Close() + defer close(stopCh) + + providerKey := types.NamespacedName{ + Name: fmt.Sprintf("provider-%s", randStringRunes(5)), + Namespace: namespace, + } + provider = &apiv1.Provider{ + ObjectMeta: metav1.ObjectMeta{ + Name: providerKey.Name, + Namespace: providerKey.Namespace, + }, + Spec: apiv1.ProviderSpec{ + Type: "generic", + Address: rcvServer.URL, + }, + } + g.Expect(k8sClient.Create(context.Background(), provider)).To(Succeed()) + g.Eventually(func() bool { + var obj apiv1.Provider + g.Expect(k8sClient.Get(context.Background(), client.ObjectKeyFromObject(provider), &obj)) + return conditions.IsReady(&obj) + }, 30*time.Second, time.Second).Should(BeTrue()) + + repo, err := readManifest("./testdata/repo.yaml", namespace) + g.Expect(err).ToNot(HaveOccurred()) + + secondRepo, err := readManifest("./testdata/gitrepo2.yaml", namespace) + g.Expect(err).ToNot(HaveOccurred()) + + _, err = manager.Apply(context.Background(), repo, ssa.ApplyOptions{ + Force: true, + }) + g.Expect(err).ToNot(HaveOccurred()) + + _, err = manager.Apply(context.Background(), secondRepo, ssa.ApplyOptions{ + Force: true, + }) + g.Expect(err).ToNot(HaveOccurred()) + + alertKey := types.NamespacedName{ + Name: fmt.Sprintf("alert-%s", randStringRunes(5)), + Namespace: namespace, + } + + alert := &apiv1.Alert{ + ObjectMeta: metav1.ObjectMeta{ + Name: alertKey.Name, + Namespace: alertKey.Namespace, + }, + Spec: apiv1.AlertSpec{ + ProviderRef: meta.LocalObjectReference{ + Name: providerKey.Name, + }, + EventSeverity: "info", + EventSources: []apiv1.CrossNamespaceObjectReference{ + { + Kind: "Bucket", + Name: "hyacinth", + Namespace: namespace, + }, + { + Kind: "Bucket", + Name: "daffodil", + Namespace: "test", + }, + { + Kind: "Bucket", + Name: "bluebells", + Namespace: "*", + }, + { + Kind: "Kustomization", + Name: "*", + Namespace: namespace, + }, + { + Kind: "HelmRelease", + Name: "*", + Namespace: "*", + }, + }, + ExclusionList: []string{ + "doesnotoccur", // not intended to match + "excluded", + }, + }, + } + + g.Expect(k8sClient.Create(context.Background(), alert)).To(Succeed()) + + // wait for controller to mark the alert as ready + g.Eventually(func() bool { + var obj apiv1.Alert + g.Expect(k8sClient.Get(context.Background(), client.ObjectKeyFromObject(alert), &obj)) + return conditions.IsReady(&obj) + }, 30*time.Second, time.Second).Should(BeTrue()) + + event := eventv1.Event{ + InvolvedObject: corev1.ObjectReference{ + Kind: "Bucket", + Name: "hyacinth", + Namespace: namespace, + }, + Severity: "info", + Timestamp: metav1.Now(), + Message: "well that happened", + Reason: "event-happened", + ReportingController: "source-controller", + } + + testSent := func() { + buf := &bytes.Buffer{} + g.Expect(json.NewEncoder(buf).Encode(&event)).To(Succeed()) + res, err := http.Post("http://localhost:56789/", "application/json", buf) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(res.StatusCode).To(Equal(202)) // event_server responds with 202 Accepted + } + + testForwarded := func() { + g.Eventually(func() bool { + return req == nil + }, "2s", "0.1s").Should(BeFalse()) + } + + testFiltered := func() { + // The event_server does forwarding in a goroutine, after + // responding to the POST of the event. This makes it + // difficult to know whether the provider has filtered the + // event, or just not run the goroutine yet. For now, I'll use + // a timeout (and Consistently so it can fail early) + g.Consistently(func() bool { + return req == nil + }, "1s", "0.1s").Should(BeTrue()) + } + + tests := []struct { + name string + modifyEventFunc func(e eventv1.Event) eventv1.Event + forwarded bool + }{ + { + name: "forwards when source is a match", + modifyEventFunc: func(e eventv1.Event) eventv1.Event { return e }, + forwarded: true, + }, + { + name: "forward events for cross-namespace sources", + modifyEventFunc: func(e eventv1.Event) eventv1.Event { + e.InvolvedObject.Kind = "Bucket" + e.InvolvedObject.Name = "daffodil" + e.InvolvedObject.Namespace = "test" + return e + }, + forwarded: true, + }, + { + name: "forwards events when the namespace wildcard is used", + modifyEventFunc: func(e eventv1.Event) eventv1.Event { + e.InvolvedObject.Kind = "Bucket" + e.InvolvedObject.Name = "bluebells" + e.InvolvedObject.Namespace = "test-" + randStringRunes(5) + return e + }, + forwarded: true, + }, + { + name: "drops event when source Kind does not match", + modifyEventFunc: func(e eventv1.Event) eventv1.Event { + e.InvolvedObject.Kind = "GitRepository" + e.InvolvedObject.Namespace = namespace + return e + }, + forwarded: false, + }, + { + name: "drops event when source name does not match", + modifyEventFunc: func(e eventv1.Event) eventv1.Event { + e.InvolvedObject.Kind = "Bucket" + e.InvolvedObject.Name = "slop" + e.InvolvedObject.Namespace = namespace + return e + }, + forwarded: false, + }, + { + name: "drops event when source namespace does not match", + modifyEventFunc: func(e eventv1.Event) eventv1.Event { + e.InvolvedObject.Kind = "Bucket" + e.InvolvedObject.Name = "hyacinth" + e.InvolvedObject.Namespace = "all-buckets" + return e + }, + forwarded: false, + }, + { + name: "drops event that is matched by exclusion", + modifyEventFunc: func(e eventv1.Event) eventv1.Event { + e.InvolvedObject.Kind = "Bucket" + e.InvolvedObject.Name = "bluebells" + e.InvolvedObject.Namespace = "test-" + randStringRunes(5) + e.Message = "this is excluded" + return e + }, + forwarded: false, + }, + { + name: "forwards events when name wildcard is used", + modifyEventFunc: func(e eventv1.Event) eventv1.Event { + e.InvolvedObject.Kind = "Kustomization" + e.InvolvedObject.Name = "test" + e.InvolvedObject.Namespace = namespace + e.Message = "test" + return e + }, + forwarded: true, + }, + { + name: "forwards events when name and namespace wildcards is used", + modifyEventFunc: func(e eventv1.Event) eventv1.Event { + e.InvolvedObject.Kind = "HelmRelease" + e.InvolvedObject.Name = "test" + e.InvolvedObject.Namespace = "test-" + randStringRunes(5) + e.Message = "test" + return e + }, + forwarded: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + event = tt.modifyEventFunc(event) + testSent() + if tt.forwarded { + testForwarded() + } else { + testFiltered() + } + req = nil + }) + } +} From b2444e7fc5534a42b7b7abe906ade63c4aa66188 Mon Sep 17 00:00:00 2001 From: alekspog Date: Thu, 30 Mar 2023 22:06:16 +0300 Subject: [PATCH 5/9] use custom controller registry in tests Signed-off-by: alekspog --- go.mod | 2 +- internal/controllers/alert_controller_test.go | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 5c278f7ba..28a55fa33 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/ktrysmt/go-bitbucket v0.9.55 github.com/microsoft/azure-devops-go-api/azuredevops/v6 v6.0.1 github.com/onsi/gomega v1.27.2 + github.com/prometheus/client_golang v1.14.0 github.com/sethvargo/go-limiter v0.7.2 github.com/slok/go-http-metrics v0.10.0 github.com/spf13/pflag v1.0.5 @@ -110,7 +111,6 @@ require ( github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_golang v1.14.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect diff --git a/internal/controllers/alert_controller_test.go b/internal/controllers/alert_controller_test.go index 0d651828b..1ac059db7 100644 --- a/internal/controllers/alert_controller_test.go +++ b/internal/controllers/alert_controller_test.go @@ -28,6 +28,7 @@ import ( "github.com/fluxcd/pkg/ssa" . "github.com/onsi/gomega" + "github.com/prometheus/client_golang/prometheus" "github.com/sethvargo/go-limiter/memorystore" prommetrics "github.com/slok/go-http-metrics/metrics/prometheus" "github.com/slok/go-http-metrics/middleware" @@ -169,9 +170,12 @@ func TestAlertReconciler_EventHandler(t *testing.T) { ) g.Expect(createNamespace(namespace)).NotTo(HaveOccurred(), "failed to create test namespace") + reg := prometheus.NewRegistry() + eventMdlw := middleware.New(middleware.Config{ Recorder: prommetrics.NewRecorder(prommetrics.Config{ - Prefix: "gotk_event", + Prefix: "gotk_event", + Registry: reg, }), }) @@ -435,9 +439,12 @@ func TestAlertReconciler_EventHandler_CrossNamespaceRefs(t *testing.T) { ) g.Expect(createNamespace(namespace)).NotTo(HaveOccurred(), "failed to create test namespace") + reg := prometheus.NewRegistry() + eventMdlw := middleware.New(middleware.Config{ Recorder: prommetrics.NewRecorder(prommetrics.Config{ - Prefix: "gotk_event", + Prefix: "gotk_event", + Registry: reg, }), }) From 99c6b5133480c93afeb116cbbff4763795486059 Mon Sep 17 00:00:00 2001 From: alekspog Date: Fri, 24 Mar 2023 13:54:42 +0200 Subject: [PATCH 6/9] add controller test Signed-off-by: alekspog --- internal/controllers/alert_controller_test.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/internal/controllers/alert_controller_test.go b/internal/controllers/alert_controller_test.go index 1ac059db7..127717195 100644 --- a/internal/controllers/alert_controller_test.go +++ b/internal/controllers/alert_controller_test.go @@ -271,6 +271,11 @@ func TestAlertReconciler_EventHandler(t *testing.T) { Name: "*", Namespace: "test", }, + { + Kind: "Kustomization", + Name: "testwildcardnamespace", + Namespace: "*", + }, }, ExclusionList: []string{ "doesnotoccur", // not intended to match @@ -379,6 +384,17 @@ func TestAlertReconciler_EventHandler(t *testing.T) { }, forwarded: true, }, + { + name: "forwards events when namespace wildcard is used", + modifyEventFunc: func(e eventv1.Event) eventv1.Event { + e.InvolvedObject.Kind = "Kustomization" + e.InvolvedObject.Name = "testwildcardnamespace" + e.InvolvedObject.Namespace = "test-" + randStringRunes(5) + e.Message = "test" + return e + }, + forwarded: true, + }, { name: "forwards events when the label matches", modifyEventFunc: func(e eventv1.Event) eventv1.Event { From bc08e1ac7127ae741c31a0f7b661bde03032b433 Mon Sep 17 00:00:00 2001 From: alekspog Date: Thu, 30 Mar 2023 21:22:45 +0300 Subject: [PATCH 7/9] rebase on main Signed-off-by: alekspog --- internal/controllers/alert_controller_test.go | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/internal/controllers/alert_controller_test.go b/internal/controllers/alert_controller_test.go index 127717195..1ac059db7 100644 --- a/internal/controllers/alert_controller_test.go +++ b/internal/controllers/alert_controller_test.go @@ -271,11 +271,6 @@ func TestAlertReconciler_EventHandler(t *testing.T) { Name: "*", Namespace: "test", }, - { - Kind: "Kustomization", - Name: "testwildcardnamespace", - Namespace: "*", - }, }, ExclusionList: []string{ "doesnotoccur", // not intended to match @@ -384,17 +379,6 @@ func TestAlertReconciler_EventHandler(t *testing.T) { }, forwarded: true, }, - { - name: "forwards events when namespace wildcard is used", - modifyEventFunc: func(e eventv1.Event) eventv1.Event { - e.InvolvedObject.Kind = "Kustomization" - e.InvolvedObject.Name = "testwildcardnamespace" - e.InvolvedObject.Namespace = "test-" + randStringRunes(5) - e.Message = "test" - return e - }, - forwarded: true, - }, { name: "forwards events when the label matches", modifyEventFunc: func(e eventv1.Event) eventv1.Event { From 371ea46b1ea28a2c482cb9cefecfdd506bf0c823 Mon Sep 17 00:00:00 2001 From: alekspog Date: Thu, 30 Mar 2023 22:56:44 +0300 Subject: [PATCH 8/9] bump api version Signed-off-by: alekspog --- internal/controllers/alert_controller_test.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/internal/controllers/alert_controller_test.go b/internal/controllers/alert_controller_test.go index 1ac059db7..780f25723 100644 --- a/internal/controllers/alert_controller_test.go +++ b/internal/controllers/alert_controller_test.go @@ -435,7 +435,7 @@ func TestAlertReconciler_EventHandler_CrossNamespaceRefs(t *testing.T) { var ( namespace = "events-" + randStringRunes(5) req *http.Request - provider *apiv1.Provider + provider *apiv1beta2.Provider ) g.Expect(createNamespace(namespace)).NotTo(HaveOccurred(), "failed to create test namespace") @@ -470,19 +470,19 @@ func TestAlertReconciler_EventHandler_CrossNamespaceRefs(t *testing.T) { Name: fmt.Sprintf("provider-%s", randStringRunes(5)), Namespace: namespace, } - provider = &apiv1.Provider{ + provider = &apiv1beta2.Provider{ ObjectMeta: metav1.ObjectMeta{ Name: providerKey.Name, Namespace: providerKey.Namespace, }, - Spec: apiv1.ProviderSpec{ + Spec: apiv1beta2.ProviderSpec{ Type: "generic", Address: rcvServer.URL, }, } g.Expect(k8sClient.Create(context.Background(), provider)).To(Succeed()) g.Eventually(func() bool { - var obj apiv1.Provider + var obj apiv1beta2.Provider g.Expect(k8sClient.Get(context.Background(), client.ObjectKeyFromObject(provider), &obj)) return conditions.IsReady(&obj) }, 30*time.Second, time.Second).Should(BeTrue()) @@ -508,12 +508,12 @@ func TestAlertReconciler_EventHandler_CrossNamespaceRefs(t *testing.T) { Namespace: namespace, } - alert := &apiv1.Alert{ + alert := &apiv1beta2.Alert{ ObjectMeta: metav1.ObjectMeta{ Name: alertKey.Name, Namespace: alertKey.Namespace, }, - Spec: apiv1.AlertSpec{ + Spec: apiv1beta2.AlertSpec{ ProviderRef: meta.LocalObjectReference{ Name: providerKey.Name, }, @@ -556,7 +556,7 @@ func TestAlertReconciler_EventHandler_CrossNamespaceRefs(t *testing.T) { // wait for controller to mark the alert as ready g.Eventually(func() bool { - var obj apiv1.Alert + var obj apiv1beta2.Alert g.Expect(k8sClient.Get(context.Background(), client.ObjectKeyFromObject(alert), &obj)) return conditions.IsReady(&obj) }, 30*time.Second, time.Second).Should(BeTrue()) From 1898b4dd63ffe5f411eb8f24b92ab9472230600d Mon Sep 17 00:00:00 2001 From: alekspog Date: Thu, 30 Mar 2023 23:02:34 +0300 Subject: [PATCH 9/9] add documentation Signed-off-by: alekspog --- docs/spec/v1beta2/alerts.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/spec/v1beta2/alerts.md b/docs/spec/v1beta2/alerts.md index 4bdd8d89a..f600e0c99 100644 --- a/docs/spec/v1beta2/alerts.md +++ b/docs/spec/v1beta2/alerts.md @@ -142,6 +142,19 @@ eventSources: namespace: apps ``` +#### Select objects from all namespaces + +The `*` wildcard can be used to select events issued by Flux objects from any `namespace`: + +```yaml +eventSources: + - kind: HelmRelease + name: 'service1' + namespace: '*' +``` + +It requires to have cross-namespace references enabled for the controller. + #### Select objects by label To select events issued by all Flux objects of a particular `kind` with specific `labels`: