Runtime, Recorder: Use httpClient.CheckRetry to handle http code 429 Too Many Requests.

The notification controller sends a 429 Too Many Requests message when a message is duplicated. Thus if we
receive a 429, we should discard the message.

Author: jvcdk

runtime/events/recorder.go

@@ -17,7 +17,7 @@ limitations under the License.
 package events
 
 import (
-	"bytes"
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -93,7 +93,7 @@ func NewRecorder(mgr ctrl.Manager, log logr.Logger, webhook, reportingController
 
 	httpClient := retryablehttp.NewClient()
 	httpClient.HTTPClient.Timeout = 5 * time.Second
-	httpClient.CheckRetry = retryablehttp.ErrorPropagatedRetryPolicy
+	httpClient.CheckRetry = checkRetry
 	httpClient.Logger = nil
 
 	return &Recorder{
@@ -120,7 +120,7 @@ func NewRecorderForScheme(scheme *runtime.Scheme,
 
 	httpClient := retryablehttp.NewClient()
 	httpClient.HTTPClient.Timeout = 5 * time.Second
-	httpClient.CheckRetry = retryablehttp.ErrorPropagatedRetryPolicy
+	httpClient.CheckRetry = checkRetry
 	httpClient.Logger = nil
 
 	return &Recorder{
@@ -133,6 +133,19 @@ func NewRecorderForScheme(scheme *runtime.Scheme,
 	}, nil
 }
 
+func checkRetry(ctx context.Context, resp *http.Response, err error) (bool, error) {
+	if resp != nil && responseIsMessageIsDuplicated(resp) {
+		return false, nil // Don't retry
+	}
+	return retryablehttp.ErrorPropagatedRetryPolicy(ctx, resp, err)
+}
+
+// The Notification Controller returns a 429 Too Many Requests response
+// when the posted message is a duplicate (within a certain time window).
+func responseIsMessageIsDuplicated(resp *http.Response) bool {
+	return resp.StatusCode == http.StatusTooManyRequests
+}
+
 // Event records an event in the webhook address.
 func (r *Recorder) Event(object runtime.Object, eventtype, reason, message string) {
 	r.AnnotatedEventf(object, nil, eventtype, reason, "%s", message)
@@ -250,12 +263,6 @@ func (r *Recorder) AnnotatedEventf(
 		return
 	}
 
-	// avoid retrying rate limited requests
-	if res, _ := r.Client.HTTPClient.Post(r.Webhook, "application/json", bytes.NewReader(body)); res != nil &&
-		(res.StatusCode == http.StatusTooManyRequests || res.StatusCode == http.StatusAccepted) {
-		return
-	}
-
 	if _, err := r.Client.Post(r.Webhook, "application/json", body); err != nil {
 		log.Error(err, "unable to record event")
 		return