diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java index 17046d2f6cfb..c88232eb357e 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java @@ -34,23 +34,31 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.logging.Level; +import java.util.logging.Logger; import org.jspecify.annotations.NullMarked; /** * A composite implementation of {@link ApiTracer} that delegates all tracing events to a list of * underlying tracers. * - *
Completion and failure callbacks execute asynchronously on background worker threads where no - * trace span is active on thread-local storage. Wrapping those callbacks in {@code try (Scope - * ignored = inScope())} briefly activates the current span on the callback thread before any child - * tracer ends it, allowing sibling tracers (such as logging and metrics) to correlate error logs - * and metric exemplars with the active span before the scope closes on the same thread. + *
Attempt and operation completion callbacks often run on a different thread than the one that + * started the attempt, where no span is current. They are dispatched inside {@link #inScope()} so + * that sibling tracers (such as logging and metrics) observe the span activated by a tracing child + * before that child ends it. Entering and exiting that scope is best-effort: these callbacks are + * invoked from GAX's retry and future-completion machinery, where an exception would skip the + * remaining callbacks or prevent the RPC future from completing. * *
For internal use only.
*/
@NullMarked
@InternalApi
class CompositeTracer extends BaseApiTracer {
+ private static final Logger LOGGER = Logger.getLogger(CompositeTracer.class.getName());
+ private static final AtomicBoolean SCOPE_FAILURE_WARNED = new AtomicBoolean();
+ private static final Scope NOOP_CALLBACK_SCOPE = () -> {};
+
private final List Completion callbacks are invoked from {@code BasicRetryingFuture.handleAttempt} and from
+ * future listeners. If entering or closing the scope threw there, the remaining callbacks would
+ * be skipped and, on the attempt path, the RPC future would never complete. Failures are logged
+ * and the callbacks are dispatched without the scope instead.
+ */
+ private Scope enterCallbackScope() {
+ final Scope scope;
+ try {
+ scope = inScope();
+ } catch (RuntimeException e) {
+ logScopeFailure("Failed to enter tracer scope; dispatching callbacks without it", e);
+ return NOOP_CALLBACK_SCOPE;
+ }
+ return () -> {
+ try {
+ scope.close();
+ } catch (RuntimeException e) {
+ logScopeFailure("Failed to close tracer scope", e);
+ }
+ };
+ }
+
+ /**
+ * Logs a best-effort scope failure at {@link Level#WARNING} once, and at {@link Level#FINE}
+ * thereafter to avoid log flooding.
+ */
+ private static void logScopeFailure(String message, RuntimeException e) {
+ Level level = SCOPE_FAILURE_WARNED.compareAndSet(false, true) ? Level.WARNING : Level.FINE;
+ LOGGER.log(level, message, e);
+ }
+
@Override
public void operationSucceeded() {
- try (Scope ignored = inScope()) {
+ try (Scope ignored = enterCallbackScope()) {
for (int i = children.size() - 1; i >= 0; i--) {
children.get(i).operationSucceeded();
}
@@ -115,7 +157,7 @@ public void operationSucceeded() {
@Override
public void operationCancelled() {
- try (Scope ignored = inScope()) {
+ try (Scope ignored = enterCallbackScope()) {
for (int i = children.size() - 1; i >= 0; i--) {
children.get(i).operationCancelled();
}
@@ -124,7 +166,7 @@ public void operationCancelled() {
@Override
public void operationFailed(Throwable error) {
- try (Scope ignored = inScope()) {
+ try (Scope ignored = enterCallbackScope()) {
for (int i = children.size() - 1; i >= 0; i--) {
children.get(i).operationFailed(error);
}
@@ -155,7 +197,7 @@ public void attemptStarted(Object request, int attemptNumber) {
@Override
public void attemptSucceeded() {
- try (Scope ignored = inScope()) {
+ try (Scope ignored = enterCallbackScope()) {
for (int i = children.size() - 1; i >= 0; i--) {
children.get(i).attemptSucceeded();
}
@@ -164,7 +206,7 @@ public void attemptSucceeded() {
@Override
public void attemptCancelled() {
- try (Scope ignored = inScope()) {
+ try (Scope ignored = enterCallbackScope()) {
for (int i = children.size() - 1; i >= 0; i--) {
children.get(i).attemptCancelled();
}
@@ -173,7 +215,7 @@ public void attemptCancelled() {
@Override
public void attemptFailed(Throwable error, org.threeten.bp.Duration delay) {
- try (Scope ignored = inScope()) {
+ try (Scope ignored = enterCallbackScope()) {
for (int i = children.size() - 1; i >= 0; i--) {
children.get(i).attemptFailed(error, delay);
}
@@ -182,7 +224,7 @@ public void attemptFailed(Throwable error, org.threeten.bp.Duration delay) {
@Override
public void attemptFailedDuration(Throwable error, java.time.Duration delay) {
- try (Scope ignored = inScope()) {
+ try (Scope ignored = enterCallbackScope()) {
for (int i = children.size() - 1; i >= 0; i--) {
children.get(i).attemptFailedDuration(error, delay);
}
@@ -191,7 +233,7 @@ public void attemptFailedDuration(Throwable error, java.time.Duration delay) {
@Override
public void attemptFailedRetriesExhausted(Throwable error) {
- try (Scope ignored = inScope()) {
+ try (Scope ignored = enterCallbackScope()) {
for (int i = children.size() - 1; i >= 0; i--) {
children.get(i).attemptFailedRetriesExhausted(error);
}
@@ -200,7 +242,7 @@ public void attemptFailedRetriesExhausted(Throwable error) {
@Override
public void attemptPermanentFailure(Throwable error) {
- try (Scope ignored = inScope()) {
+ try (Scope ignored = enterCallbackScope()) {
for (int i = children.size() - 1; i >= 0; i--) {
children.get(i).attemptPermanentFailure(error);
}
diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/RetryingTracerScopeFailureTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/RetryingTracerScopeFailureTest.java
new file mode 100644
index 000000000000..a29ea41376b2
--- /dev/null
+++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/RetryingTracerScopeFailureTest.java
@@ -0,0 +1,134 @@
+/*
+ * Copyright 2026 Google LLC
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Google LLC nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package com.google.api.gax.rpc;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import com.google.api.core.ApiFuture;
+import com.google.api.core.ApiFutures;
+import com.google.api.core.NanoClock;
+import com.google.api.gax.retrying.RetrySettings;
+import com.google.api.gax.rpc.testing.FakeCallContext;
+import com.google.api.gax.rpc.testing.FakeStatusCode;
+import com.google.api.gax.tracing.ApiTracer;
+import com.google.api.gax.tracing.ApiTracerFactory;
+import com.google.api.gax.tracing.BaseApiTracer;
+import com.google.api.gax.tracing.BaseApiTracerFactory;
+import com.google.api.gax.tracing.CompositeTracerFactory;
+import com.google.api.gax.tracing.SpanName;
+import com.google.api.gax.tracing.TracedUnaryCallable;
+import com.google.common.collect.ImmutableList;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+/**
+ * End-to-end regression test verifying that a retrying traced RPC future completes with the
+ * original RPC error instead of hanging when a child tracer's {@link ApiTracer#inScope()} throws.
+ */
+class RetryingTracerScopeFailureTest {
+ private ScheduledExecutorService executor;
+ private ClientContext clientContext;
+
+ /** Initializes a scheduled executor and {@link ClientContext} for retrying callable tests. */
+ @BeforeEach
+ void setUp() {
+ executor = Executors.newSingleThreadScheduledExecutor();
+ clientContext =
+ ClientContext.newBuilder()
+ .setExecutor(executor)
+ .setClock(NanoClock.getDefaultClock())
+ .setDefaultCallContext(FakeCallContext.createDefault())
+ .build();
+ }
+
+ /** Shuts down the test executor. */
+ @AfterEach
+ void tearDown() {
+ executor.shutdownNow();
+ }
+
+ /**
+ * Verifies that the retrying future completes immediately with the original RPC error even when a
+ * child tracer's {@link ApiTracer#inScope()} throws during completion callbacks.
+ */
+ @Test
+ void retryingFutureCompletes_whenTracerScopeThrows() {
+ ApiException rpcError =
+ new InvalidArgumentException(
+ null, FakeStatusCode.of(StatusCode.Code.INVALID_ARGUMENT), false);
+
+ UnaryCallable