From be20640ae457068ca7492ff10e5059d8c80dab81 Mon Sep 17 00:00:00 2001 From: Surbhi Garg Date: Fri, 14 Nov 2025 12:02:11 +0530 Subject: [PATCH] fix: Connectivity Error Metrics --- .../spanner/SpannerGrpcStreamTracer.java | 56 +++++++++++++++++++ .../spanner/spi/v1/HeaderInterceptor.java | 31 +++++++--- ...OpenTelemetryBuiltInMetricsTracerTest.java | 4 ++ 3 files changed, 83 insertions(+), 8 deletions(-) create mode 100644 google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerGrpcStreamTracer.java diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerGrpcStreamTracer.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerGrpcStreamTracer.java new file mode 100644 index 0000000000..b3ad18c975 --- /dev/null +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/SpannerGrpcStreamTracer.java @@ -0,0 +1,56 @@ +/* + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.cloud.spanner; + +import io.grpc.ClientStreamTracer; +import io.grpc.Metadata; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * Captures the event when a request is sent from the gRPC client. Its primary purpose is to measure + * the transition time between asking gRPC to start an RPC and gRPC actually serializing that RPC. + */ +public class SpannerGrpcStreamTracer extends ClientStreamTracer { + + private final AtomicBoolean outBoundMessageSent = new AtomicBoolean(false); + + public SpannerGrpcStreamTracer() {} + + public boolean isOutBoundMessageSent() { + return outBoundMessageSent.get(); + } + + /** An outbound message has been serialized and sent to the transport. */ + @Override + public void outboundMessageSent(int seqNo, long optionalWireSize, long optionalUncompressedSize) { + outBoundMessageSent.set(true); + } + + public static class Factory extends ClientStreamTracer.Factory { + + SpannerGrpcStreamTracer spannerGrpcStreamTracer; + + public Factory(SpannerGrpcStreamTracer spannerGrpcStreamTracer) { + this.spannerGrpcStreamTracer = spannerGrpcStreamTracer; + } + + @Override + public ClientStreamTracer newClientStreamTracer( + ClientStreamTracer.StreamInfo info, Metadata headers) { + return spannerGrpcStreamTracer; + } + } +} diff --git a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/HeaderInterceptor.java b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/HeaderInterceptor.java index 1aaa7bd304..7152774a4f 100644 --- a/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/HeaderInterceptor.java +++ b/google-cloud-spanner/src/main/java/com/google/cloud/spanner/spi/v1/HeaderInterceptor.java @@ -28,15 +28,9 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import com.google.spanner.admin.database.v1.DatabaseName; -import io.grpc.CallOptions; -import io.grpc.Channel; -import io.grpc.ClientCall; -import io.grpc.ClientInterceptor; +import io.grpc.*; import io.grpc.ForwardingClientCall.SimpleForwardingClientCall; import io.grpc.ForwardingClientCallListener.SimpleForwardingClientCallListener; -import io.grpc.Grpc; -import io.grpc.Metadata; -import io.grpc.MethodDescriptor; import io.opencensus.stats.MeasureMap; import io.opencensus.stats.Stats; import io.opencensus.stats.StatsRecorder; @@ -53,6 +47,7 @@ import java.util.HashMap; import java.util.Map; import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.logging.Level; import java.util.logging.Logger; import java.util.regex.Matcher; @@ -103,9 +98,13 @@ class HeaderInterceptor implements ClientInterceptor { public ClientCall interceptCall( MethodDescriptor method, CallOptions callOptions, Channel next) { ApiTracer tracer = callOptions.getOption(TRACER_KEY); + SpannerGrpcStreamTracer streamTracer = new SpannerGrpcStreamTracer(); + CallOptions newOptions = + callOptions.withStreamTracerFactory(new SpannerGrpcStreamTracer.Factory(streamTracer)); CompositeTracer compositeTracer = tracer instanceof CompositeTracer ? (CompositeTracer) tracer : null; - return new SimpleForwardingClientCall(next.newCall(method, callOptions)) { + final AtomicBoolean headersReceived = new AtomicBoolean(false); + return new SimpleForwardingClientCall(next.newCall(method, newOptions)) { @Override public void start(Listener responseListener, Metadata headers) { try { @@ -127,6 +126,7 @@ public void start(Listener responseListener, Metadata headers) { new SimpleForwardingClientCallListener(responseListener) { @Override public void onHeaders(Metadata metadata) { + headersReceived.set(true); Boolean isDirectPathUsed = isDirectPathUsed(getAttributes().get(Grpc.TRANSPORT_ATTR_REMOTE_ADDR)); addDirectPathUsedAttribute(compositeTracer, isDirectPathUsed); @@ -135,6 +135,21 @@ public void onHeaders(Metadata metadata) { metadata, tagContext, attributes, span, compositeTracer, isDirectPathUsed); super.onHeaders(metadata); } + + @Override + public void onClose(Status status, Metadata trailers) { + if (streamTracer.isOutBoundMessageSent() && !headersReceived.get()) { + // RPC was sent, but no response headers were received. This can happen in + // case of a timeout, for example. + if (compositeTracer != null) { + compositeTracer.recordGfeHeaderMissingCount(1L); + if (GapicSpannerRpc.isEnableAFEServerTiming()) { + // compositeTracer.recordAfeHeaderMissingCount(1L); + } + } + } + super.onClose(status, trailers); + } }, headers); } catch (ExecutionException executionException) { diff --git a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/OpenTelemetryBuiltInMetricsTracerTest.java b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/OpenTelemetryBuiltInMetricsTracerTest.java index aeb9487e2e..e34585c3e8 100644 --- a/google-cloud-spanner/src/test/java/com/google/cloud/spanner/OpenTelemetryBuiltInMetricsTracerTest.java +++ b/google-cloud-spanner/src/test/java/com/google/cloud/spanner/OpenTelemetryBuiltInMetricsTracerTest.java @@ -347,6 +347,10 @@ public void testNoNetworkConnection() { // Attempt count should have a failed metric point for CreateSession. assertEquals( 1, getAggregatedValue(attemptCountMetricData, expectedAttributesCreateSessionFailed), 0); + + // Connectivity count will not increase as client did not attempt to send the request + assertFalse( + checkIfMetricExists(metricReader, BuiltInMetricsConstant.GFE_CONNECTIVITY_ERROR_NAME)); } @Test