Skip to content

Commit 9d4a628

Browse files
authored
Add logging for endpoint discovery issues (#3667)
Problem: A couple of issues have arisen relating to endpoint discovery, and we don't have much visibility into them. Solution: Add some logs (both error and debug level) to help diagnose when endpoints are not found or have issues.
1 parent a847da1 commit 9d4a628

File tree

8 files changed

+127
-42
lines changed

8 files changed

+127
-42
lines changed

internal/controller/handler.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -208,7 +208,7 @@ func (h *eventHandlerImpl) sendNginxConfig(ctx context.Context, logger logr.Logg
208208
panic("expected deployment, got nil")
209209
}
210210

211-
cfg := dataplane.BuildConfiguration(ctx, gr, gw, h.cfg.serviceResolver, h.cfg.plus)
211+
cfg := dataplane.BuildConfiguration(ctx, logger, gr, gw, h.cfg.serviceResolver, h.cfg.plus)
212212
depCtx, getErr := h.getDeploymentContext(ctx)
213213
if getErr != nil {
214214
logger.Error(getErr, "error getting deployment context for usage reporting")

internal/controller/nginx/agent/agent.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,7 @@ func (n *NginxUpdaterImpl) UpdateConfig(
9090
) {
9191
msg := deployment.SetFiles(files)
9292
if msg == nil {
93+
n.logger.V(1).Info("No changes to nginx configuration files, not sending to agent")
9394
return
9495
}
9596

internal/controller/state/dataplane/configuration.go

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"slices"
88
"sort"
99

10+
"github.com/go-logr/logr"
1011
discoveryV1 "k8s.io/api/discovery/v1"
1112
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1213
"k8s.io/apimachinery/pkg/types"
@@ -32,6 +33,7 @@ const (
3233
// BuildConfiguration builds the Configuration from the Graph.
3334
func BuildConfiguration(
3435
ctx context.Context,
36+
logger logr.Logger,
3537
g *graph.Graph,
3638
gateway *graph.Gateway,
3739
serviceResolver resolver.ServiceResolver,
@@ -55,6 +57,7 @@ func BuildConfiguration(
5557
backendGroups := buildBackendGroups(append(httpServers, sslServers...))
5658
upstreams := buildUpstreams(
5759
ctx,
60+
logger,
5861
gateway,
5962
serviceResolver,
6063
g.ReferencedServices,
@@ -71,9 +74,14 @@ func BuildConfiguration(
7174
SSLServers: sslServers,
7275
TLSPassthroughServers: buildPassthroughServers(gateway),
7376
Upstreams: upstreams,
74-
StreamUpstreams: buildStreamUpstreams(ctx, gateway, serviceResolver, baseHTTPConfig.IPFamily),
75-
BackendGroups: backendGroups,
76-
SSLKeyPairs: buildSSLKeyPairs(g.ReferencedSecrets, gateway.Listeners),
77+
StreamUpstreams: buildStreamUpstreams(
78+
ctx,
79+
logger,
80+
gateway,
81+
serviceResolver,
82+
baseHTTPConfig.IPFamily),
83+
BackendGroups: backendGroups,
84+
SSLKeyPairs: buildSSLKeyPairs(g.ReferencedSecrets, gateway.Listeners),
7785
CertBundles: buildCertBundles(
7886
buildRefCertificateBundles(g.ReferencedSecrets, g.ReferencedCaCertConfigMaps),
7987
backendGroups,
@@ -163,6 +171,7 @@ func buildPassthroughServers(gateway *graph.Gateway) []Layer4VirtualServer {
163171
// buildStreamUpstreams builds all stream upstreams.
164172
func buildStreamUpstreams(
165173
ctx context.Context,
174+
logger logr.Logger,
166175
gateway *graph.Gateway,
167176
serviceResolver resolver.ServiceResolver,
168177
ipFamily IPFamilyType,
@@ -202,7 +211,13 @@ func buildStreamUpstreams(
202211

203212
allowedAddressType := getAllowedAddressType(ipFamily)
204213

205-
eps, err := serviceResolver.Resolve(ctx, br.SvcNsName, br.ServicePort, allowedAddressType)
214+
eps, err := serviceResolver.Resolve(
215+
ctx,
216+
logger,
217+
br.SvcNsName,
218+
br.ServicePort,
219+
allowedAddressType,
220+
)
206221
if err != nil {
207222
errMsg = err.Error()
208223
}
@@ -670,6 +685,7 @@ func (hpr *hostPathRules) maxServerCount() int {
670685

671686
func buildUpstreams(
672687
ctx context.Context,
688+
logger logr.Logger,
673689
gateway *graph.Gateway,
674690
svcResolver resolver.ServiceResolver,
675691
referencedServices map[types.NamespacedName]*graph.ReferencedService,
@@ -701,6 +717,7 @@ func buildUpstreams(
701717
for _, br := range rule.BackendRefs {
702718
if upstream := buildUpstream(
703719
ctx,
720+
logger,
704721
br,
705722
gateway,
706723
svcResolver,
@@ -735,6 +752,7 @@ func buildUpstreams(
735752

736753
func buildUpstream(
737754
ctx context.Context,
755+
logger logr.Logger,
738756
br graph.BackendRef,
739757
gateway *graph.Gateway,
740758
svcResolver resolver.ServiceResolver,
@@ -760,9 +778,10 @@ func buildUpstream(
760778

761779
var errMsg string
762780

763-
eps, err := svcResolver.Resolve(ctx, br.SvcNsName, br.ServicePort, allowedAddressType)
781+
eps, err := svcResolver.Resolve(ctx, logger, br.SvcNsName, br.ServicePort, allowedAddressType)
764782
if err != nil {
765783
errMsg = err.Error()
784+
logger.Error(err, "failed to resolve endpoints", "service", br.SvcNsName)
766785
}
767786

768787
var upstreamPolicies []policies.Policy

internal/controller/state/dataplane/configuration_test.go

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"sort"
88
"testing"
99

10+
"github.com/go-logr/logr"
1011
. "github.com/onsi/gomega"
1112
apiv1 "k8s.io/api/core/v1"
1213
discoveryV1 "k8s.io/api/discovery/v1"
@@ -714,6 +715,7 @@ func TestBuildConfiguration(t *testing.T) {
714715

715716
fakeResolver.ResolveStub = func(
716717
_ context.Context,
718+
_ logr.Logger,
717719
nsName types.NamespacedName,
718720
_ apiv1.ServicePort,
719721
_ []discoveryV1.AddressType,
@@ -2533,7 +2535,8 @@ func TestBuildConfiguration(t *testing.T) {
25332535
g := NewWithT(t)
25342536

25352537
result := BuildConfiguration(
2536-
context.TODO(),
2538+
t.Context(),
2539+
logr.Discard(),
25372540
test.graph,
25382541
test.graph.Gateways[gatewayNsName],
25392542
fakeResolver,
@@ -2648,7 +2651,8 @@ func TestBuildConfiguration_Plus(t *testing.T) {
26482651
g := NewWithT(t)
26492652

26502653
result := BuildConfiguration(
2651-
context.TODO(),
2654+
t.Context(),
2655+
logr.Discard(),
26522656
test.graph,
26532657
test.graph.Gateways[gatewayNsName],
26542658
fakeResolver,
@@ -3372,6 +3376,7 @@ func TestBuildUpstreams(t *testing.T) {
33723376
fakeResolver := &resolverfakes.FakeServiceResolver{}
33733377
fakeResolver.ResolveCalls(func(
33743378
_ context.Context,
3379+
_ logr.Logger,
33753380
svcNsName types.NamespacedName,
33763381
_ apiv1.ServicePort,
33773382
_ []discoveryV1.AddressType,
@@ -3404,7 +3409,14 @@ func TestBuildUpstreams(t *testing.T) {
34043409

34053410
g := NewWithT(t)
34063411

3407-
upstreams := buildUpstreams(context.TODO(), gateway, fakeResolver, referencedServices, Dual)
3412+
upstreams := buildUpstreams(
3413+
t.Context(),
3414+
logr.Discard(),
3415+
gateway,
3416+
fakeResolver,
3417+
referencedServices,
3418+
Dual,
3419+
)
34083420
g.Expect(upstreams).To(ConsistOf(expUpstreams))
34093421
}
34103422

@@ -4357,6 +4369,7 @@ func TestBuildStreamUpstreams(t *testing.T) {
43574369

43584370
fakeResolver.ResolveStub = func(
43594371
_ context.Context,
4372+
_ logr.Logger,
43604373
nsName types.NamespacedName,
43614374
_ apiv1.ServicePort,
43624375
_ []discoveryV1.AddressType,
@@ -4367,7 +4380,7 @@ func TestBuildStreamUpstreams(t *testing.T) {
43674380
return fakeEndpoints, nil
43684381
}
43694382

4370-
streamUpstreams := buildStreamUpstreams(context.Background(), gateway, &fakeResolver, Dual)
4383+
streamUpstreams := buildStreamUpstreams(t.Context(), logr.Discard(), gateway, &fakeResolver, Dual)
43714384

43724385
expectedStreamUpstreams := []Upstream{
43734386
{

internal/controller/state/resolver/resolver.go

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"fmt"
66
"slices"
77

8+
"github.com/go-logr/logr"
89
v1 "k8s.io/api/core/v1"
910
discoveryV1 "k8s.io/api/discovery/v1"
1011
"k8s.io/apimachinery/pkg/types"
@@ -22,6 +23,7 @@ import (
2223
type ServiceResolver interface {
2324
Resolve(
2425
ctx context.Context,
26+
logger logr.Logger,
2527
svcNsName types.NamespacedName,
2628
svcPort v1.ServicePort,
2729
allowedAddressType []discoveryV1.AddressType,
@@ -52,6 +54,7 @@ func NewServiceResolverImpl(c client.Client) *ServiceResolverImpl {
5254
// Returns an error if the Service or ServicePort cannot be resolved.
5355
func (e *ServiceResolverImpl) Resolve(
5456
ctx context.Context,
57+
logger logr.Logger,
5558
svcNsName types.NamespacedName,
5659
svcPort v1.ServicePort,
5760
allowedAddressType []discoveryV1.AddressType,
@@ -76,6 +79,7 @@ func (e *ServiceResolverImpl) Resolve(
7679
}
7780

7881
return resolveEndpoints(
82+
logger,
7983
svcNsName,
8084
svcPort,
8185
endpointSliceList,
@@ -84,19 +88,23 @@ func (e *ServiceResolverImpl) Resolve(
8488
)
8589
}
8690

87-
type initEndpointSetFunc func([]discoveryV1.EndpointSlice) map[Endpoint]struct{}
91+
type initEndpointSetFunc func(logr.Logger, []discoveryV1.EndpointSlice) map[Endpoint]struct{}
8892

89-
func initEndpointSetWithCalculatedSize(endpointSlices []discoveryV1.EndpointSlice) map[Endpoint]struct{} {
93+
func initEndpointSetWithCalculatedSize(
94+
logger logr.Logger,
95+
endpointSlices []discoveryV1.EndpointSlice,
96+
) map[Endpoint]struct{} {
9097
// performance optimization to reduce the cost of growing the map. See the benchamarks for performance comparison.
91-
return make(map[Endpoint]struct{}, calculateReadyEndpoints(endpointSlices))
98+
return make(map[Endpoint]struct{}, calculateReadyEndpoints(logger, endpointSlices))
9299
}
93100

94-
func calculateReadyEndpoints(endpointSlices []discoveryV1.EndpointSlice) int {
101+
func calculateReadyEndpoints(logger logr.Logger, endpointSlices []discoveryV1.EndpointSlice) int {
95102
total := 0
96103

97104
for _, eps := range endpointSlices {
98105
for _, endpoint := range eps.Endpoints {
99106
if !endpointReady(endpoint) {
107+
logger.V(1).Info("ignoring endpoint that is not ready", "endpoint", endpoint)
100108
continue
101109
}
102110

@@ -108,6 +116,7 @@ func calculateReadyEndpoints(endpointSlices []discoveryV1.EndpointSlice) int {
108116
}
109117

110118
func resolveEndpoints(
119+
logger logr.Logger,
111120
svcNsName types.NamespacedName,
112121
svcPort v1.ServicePort,
113122
endpointSliceList discoveryV1.EndpointSliceList,
@@ -122,12 +131,13 @@ func resolveEndpoints(
122131

123132
// Endpoints may be duplicated across multiple EndpointSlices.
124133
// Using a set to prevent returning duplicate endpoints.
125-
endpointSet := initEndpointsSet(filteredSlices)
134+
endpointSet := initEndpointsSet(logger, filteredSlices)
126135

127136
for _, eps := range filteredSlices {
128137
ipv6 := eps.AddressType == discoveryV1.AddressTypeIPv6
129138
for _, endpoint := range eps.Endpoints {
130139
if !endpointReady(endpoint) {
140+
logger.V(1).Info("ignoring endpoint that is not ready", "endpoint", endpoint)
131141
continue
132142
}
133143

internal/controller/state/resolver/resolver_test.go

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"fmt"
55
"testing"
66

7+
"github.com/go-logr/logr"
78
. "github.com/onsi/gomega"
89
v1 "k8s.io/api/core/v1"
910
discoveryV1 "k8s.io/api/discovery/v1"
@@ -539,7 +540,7 @@ func TestCalculateReadyEndpoints(t *testing.T) {
539540
},
540541
}
541542

542-
result := calculateReadyEndpoints(slices)
543+
result := calculateReadyEndpoints(logr.Discard(), slices)
543544

544545
g.Expect(result).To(Equal(4))
545546
}
@@ -605,7 +606,7 @@ func BenchmarkResolve(b *testing.B) {
605606
Name: "default-name",
606607
}
607608

608-
initEndpointSet := func([]discoveryV1.EndpointSlice) map[Endpoint]struct{} {
609+
initEndpointSet := func(logr.Logger, []discoveryV1.EndpointSlice) map[Endpoint]struct{} {
609610
return make(map[Endpoint]struct{})
610611
}
611612

@@ -625,8 +626,15 @@ func bench(b *testing.B, svcNsName types.NamespacedName,
625626
list discoveryV1.EndpointSliceList, initSet initEndpointSetFunc, n int,
626627
) {
627628
b.Helper()
628-
for range b.N {
629-
res, err := resolveEndpoints(svcNsName, v1.ServicePort{Port: 80}, list, initSet, dualAddressType)
629+
for b.Loop() {
630+
res, err := resolveEndpoints(
631+
logr.Discard(),
632+
svcNsName,
633+
v1.ServicePort{Port: 80},
634+
list,
635+
initSet,
636+
dualAddressType,
637+
)
630638
if len(res) != n {
631639
b.Fatalf("expected %d endpoints, got %d", n, len(res))
632640
}

0 commit comments

Comments
 (0)