From b6d1803e9f076e74eca6414efaca161d57250177 Mon Sep 17 00:00:00 2001 From: Jin Seop Kim Date: Fri, 20 Nov 2026 17:59:38 -0500 Subject: [PATCH] fix(gax): make CompositeTracer completion-callback scoping best-effort --- .../api/gax/tracing/CompositeTracer.java | 70 +++++++-- .../rpc/RetryingTracerScopeFailureTest.java | 134 ++++++++++++++++++ .../CompositeTracerScopeFailureTest.java | 119 ++++++++++++++++ 3 files changed, 309 insertions(+), 14 deletions(-) create mode 100644 sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/RetryingTracerScopeFailureTest.java create mode 100644 sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerScopeFailureTest.java diff --git a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java index 17046d2f6cfb..c88232eb357e 100644 --- a/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java +++ b/sdk-platform-java/gax-java/gax/src/main/java/com/google/api/gax/tracing/CompositeTracer.java @@ -34,23 +34,31 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.logging.Level; +import java.util.logging.Logger; import org.jspecify.annotations.NullMarked; /** * A composite implementation of {@link ApiTracer} that delegates all tracing events to a list of * underlying tracers. * - *

Completion and failure callbacks execute asynchronously on background worker threads where no - * trace span is active on thread-local storage. Wrapping those callbacks in {@code try (Scope - * ignored = inScope())} briefly activates the current span on the callback thread before any child - * tracer ends it, allowing sibling tracers (such as logging and metrics) to correlate error logs - * and metric exemplars with the active span before the scope closes on the same thread. + *

Attempt and operation completion callbacks often run on a different thread than the one that + * started the attempt, where no span is current. They are dispatched inside {@link #inScope()} so + * that sibling tracers (such as logging and metrics) observe the span activated by a tracing child + * before that child ends it. Entering and exiting that scope is best-effort: these callbacks are + * invoked from GAX's retry and future-completion machinery, where an exception would skip the + * remaining callbacks or prevent the RPC future from completing. * *

For internal use only. */ @NullMarked @InternalApi class CompositeTracer extends BaseApiTracer { + private static final Logger LOGGER = Logger.getLogger(CompositeTracer.class.getName()); + private static final AtomicBoolean SCOPE_FAILURE_WARNED = new AtomicBoolean(); + private static final Scope NOOP_CALLBACK_SCOPE = () -> {}; + private final List children; public CompositeTracer(List children) { @@ -104,9 +112,43 @@ public Scope inScope() { }; } + /** + * Best-effort {@link #inScope()} for completion callbacks. + * + *

Completion callbacks are invoked from {@code BasicRetryingFuture.handleAttempt} and from + * future listeners. If entering or closing the scope threw there, the remaining callbacks would + * be skipped and, on the attempt path, the RPC future would never complete. Failures are logged + * and the callbacks are dispatched without the scope instead. + */ + private Scope enterCallbackScope() { + final Scope scope; + try { + scope = inScope(); + } catch (RuntimeException e) { + logScopeFailure("Failed to enter tracer scope; dispatching callbacks without it", e); + return NOOP_CALLBACK_SCOPE; + } + return () -> { + try { + scope.close(); + } catch (RuntimeException e) { + logScopeFailure("Failed to close tracer scope", e); + } + }; + } + + /** + * Logs a best-effort scope failure at {@link Level#WARNING} once, and at {@link Level#FINE} + * thereafter to avoid log flooding. + */ + private static void logScopeFailure(String message, RuntimeException e) { + Level level = SCOPE_FAILURE_WARNED.compareAndSet(false, true) ? Level.WARNING : Level.FINE; + LOGGER.log(level, message, e); + } + @Override public void operationSucceeded() { - try (Scope ignored = inScope()) { + try (Scope ignored = enterCallbackScope()) { for (int i = children.size() - 1; i >= 0; i--) { children.get(i).operationSucceeded(); } @@ -115,7 +157,7 @@ public void operationSucceeded() { @Override public void operationCancelled() { - try (Scope ignored = inScope()) { + try (Scope ignored = enterCallbackScope()) { for (int i = children.size() - 1; i >= 0; i--) { children.get(i).operationCancelled(); } @@ -124,7 +166,7 @@ public void operationCancelled() { @Override public void operationFailed(Throwable error) { - try (Scope ignored = inScope()) { + try (Scope ignored = enterCallbackScope()) { for (int i = children.size() - 1; i >= 0; i--) { children.get(i).operationFailed(error); } @@ -155,7 +197,7 @@ public void attemptStarted(Object request, int attemptNumber) { @Override public void attemptSucceeded() { - try (Scope ignored = inScope()) { + try (Scope ignored = enterCallbackScope()) { for (int i = children.size() - 1; i >= 0; i--) { children.get(i).attemptSucceeded(); } @@ -164,7 +206,7 @@ public void attemptSucceeded() { @Override public void attemptCancelled() { - try (Scope ignored = inScope()) { + try (Scope ignored = enterCallbackScope()) { for (int i = children.size() - 1; i >= 0; i--) { children.get(i).attemptCancelled(); } @@ -173,7 +215,7 @@ public void attemptCancelled() { @Override public void attemptFailed(Throwable error, org.threeten.bp.Duration delay) { - try (Scope ignored = inScope()) { + try (Scope ignored = enterCallbackScope()) { for (int i = children.size() - 1; i >= 0; i--) { children.get(i).attemptFailed(error, delay); } @@ -182,7 +224,7 @@ public void attemptFailed(Throwable error, org.threeten.bp.Duration delay) { @Override public void attemptFailedDuration(Throwable error, java.time.Duration delay) { - try (Scope ignored = inScope()) { + try (Scope ignored = enterCallbackScope()) { for (int i = children.size() - 1; i >= 0; i--) { children.get(i).attemptFailedDuration(error, delay); } @@ -191,7 +233,7 @@ public void attemptFailedDuration(Throwable error, java.time.Duration delay) { @Override public void attemptFailedRetriesExhausted(Throwable error) { - try (Scope ignored = inScope()) { + try (Scope ignored = enterCallbackScope()) { for (int i = children.size() - 1; i >= 0; i--) { children.get(i).attemptFailedRetriesExhausted(error); } @@ -200,7 +242,7 @@ public void attemptFailedRetriesExhausted(Throwable error) { @Override public void attemptPermanentFailure(Throwable error) { - try (Scope ignored = inScope()) { + try (Scope ignored = enterCallbackScope()) { for (int i = children.size() - 1; i >= 0; i--) { children.get(i).attemptPermanentFailure(error); } diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/RetryingTracerScopeFailureTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/RetryingTracerScopeFailureTest.java new file mode 100644 index 000000000000..a29ea41376b2 --- /dev/null +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/rpc/RetryingTracerScopeFailureTest.java @@ -0,0 +1,134 @@ +/* + * Copyright 2026 Google LLC + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google LLC nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package com.google.api.gax.rpc; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; + +import com.google.api.core.ApiFuture; +import com.google.api.core.ApiFutures; +import com.google.api.core.NanoClock; +import com.google.api.gax.retrying.RetrySettings; +import com.google.api.gax.rpc.testing.FakeCallContext; +import com.google.api.gax.rpc.testing.FakeStatusCode; +import com.google.api.gax.tracing.ApiTracer; +import com.google.api.gax.tracing.ApiTracerFactory; +import com.google.api.gax.tracing.BaseApiTracer; +import com.google.api.gax.tracing.BaseApiTracerFactory; +import com.google.api.gax.tracing.CompositeTracerFactory; +import com.google.api.gax.tracing.SpanName; +import com.google.api.gax.tracing.TracedUnaryCallable; +import com.google.common.collect.ImmutableList; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +/** + * End-to-end regression test verifying that a retrying traced RPC future completes with the + * original RPC error instead of hanging when a child tracer's {@link ApiTracer#inScope()} throws. + */ +class RetryingTracerScopeFailureTest { + private ScheduledExecutorService executor; + private ClientContext clientContext; + + /** Initializes a scheduled executor and {@link ClientContext} for retrying callable tests. */ + @BeforeEach + void setUp() { + executor = Executors.newSingleThreadScheduledExecutor(); + clientContext = + ClientContext.newBuilder() + .setExecutor(executor) + .setClock(NanoClock.getDefaultClock()) + .setDefaultCallContext(FakeCallContext.createDefault()) + .build(); + } + + /** Shuts down the test executor. */ + @AfterEach + void tearDown() { + executor.shutdownNow(); + } + + /** + * Verifies that the retrying future completes immediately with the original RPC error even when a + * child tracer's {@link ApiTracer#inScope()} throws during completion callbacks. + */ + @Test + void retryingFutureCompletes_whenTracerScopeThrows() { + ApiException rpcError = + new InvalidArgumentException( + null, FakeStatusCode.of(StatusCode.Code.INVALID_ARGUMENT), false); + + UnaryCallable transport = + new UnaryCallable() { + @Override + public ApiFuture futureCall(Integer request, ApiCallContext context) { + return ApiFutures.immediateFailedFuture(rpcError); + } + }; + + ApiTracer brokenScopeTracer = + new BaseApiTracer() { + @Override + public Scope inScope() { + throw new IllegalStateException("broken inScope()"); + } + }; + + ApiTracerFactory tracerFactory = + new CompositeTracerFactory( + ImmutableList.of( + BaseApiTracerFactory.getInstance(), + (parent, spanName, operationType) -> brokenScopeTracer)); + + UnaryCallSettings callSettings = + UnaryCallSettings.newUnaryCallSettingsBuilder() + .setRetryableCodes(StatusCode.Code.UNAVAILABLE) + .setRetrySettings(RetrySettings.newBuilder().setMaxAttempts(3).build()) + .build(); + + UnaryCallable callable = + new TracedUnaryCallable<>( + Callables.retrying(transport, callSettings, clientContext), + tracerFactory, + SpanName.of("FakeClient", "FakeMethod")); + + ApiFuture future = callable.futureCall(1, FakeCallContext.createDefault()); + + // If the exception from inScope() escapes BasicRetryingFuture.handleAttempt, Guava's listener + // loop swallows it and the retrying future is never completed. + assertThat(future.isDone()).isTrue(); + ExecutionException e = assertThrows(ExecutionException.class, future::get); + assertThat(e).hasCauseThat().isSameInstanceAs(rpcError); + } +} diff --git a/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerScopeFailureTest.java b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerScopeFailureTest.java new file mode 100644 index 000000000000..94d34cf8ae33 --- /dev/null +++ b/sdk-platform-java/gax-java/gax/src/test/java/com/google/api/gax/tracing/CompositeTracerScopeFailureTest.java @@ -0,0 +1,119 @@ +/* + * Copyright 2026 Google LLC + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google LLC nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ +package com.google.api.gax.tracing; + +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.mockito.Mockito.withSettings; + +import java.util.Arrays; +import java.util.function.Consumer; +import java.util.stream.Stream; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +/** + * Verifies that {@link CompositeTracer} still dispatches completion callbacks to all child tracers + * when entering or closing a child's scope throws a {@link RuntimeException}. + */ +class CompositeTracerScopeFailureTest { + private static final Throwable ERROR = new RuntimeException("rpc failed"); + private static final java.time.Duration DELAY = java.time.Duration.ofMillis(10); + private static final org.threeten.bp.Duration THREETEN_DELAY = + org.threeten.bp.Duration.ofMillis(10); + + private ApiTracer child1; + private ApiTracer child2; + private CompositeTracer compositeTracer; + + /** Initializes mock child tracers and the {@link CompositeTracer} under test. */ + @BeforeEach + void setUp() { + child1 = mock(ApiTracer.class, withSettings().withoutAnnotations()); + child2 = mock(ApiTracer.class, withSettings().withoutAnnotations()); + compositeTracer = new CompositeTracer(Arrays.asList(child1, child2)); + } + + /** Returns the nine completion callbacks that {@link CompositeTracer} dispatches in a scope. */ + @SuppressWarnings("deprecation") // attemptFailed(Throwable, org.threeten.bp.Duration) + static Stream scopedCallbacks() { + return Stream.of( + callback("operationSucceeded", ApiTracer::operationSucceeded), + callback("operationCancelled", ApiTracer::operationCancelled), + callback("operationFailed", tracer -> tracer.operationFailed(ERROR)), + callback("attemptSucceeded", ApiTracer::attemptSucceeded), + callback("attemptCancelled", ApiTracer::attemptCancelled), + callback("attemptFailed", tracer -> tracer.attemptFailed(ERROR, THREETEN_DELAY)), + callback("attemptFailedDuration", tracer -> tracer.attemptFailedDuration(ERROR, DELAY)), + callback( + "attemptFailedRetriesExhausted", tracer -> tracer.attemptFailedRetriesExhausted(ERROR)), + callback("attemptPermanentFailure", tracer -> tracer.attemptPermanentFailure(ERROR))); + } + + /** Creates a named {@link Arguments} tuple for a parameterized callback test case. */ + private static Arguments callback(String name, Consumer invocation) { + return Arguments.of(name, invocation); + } + + /** Verifies all children still receive the callback when {@link ApiTracer#inScope()} throws. */ + @ParameterizedTest(name = "{0}") + @MethodSource("scopedCallbacks") + void callbackReachesAllChildren_whenEnteringScopeThrows( + String name, Consumer callback) { + when(child1.inScope()).thenThrow(new IllegalStateException("broken inScope()")); + + callback.accept(compositeTracer); // must not throw + + callback.accept(verify(child1)); + callback.accept(verify(child2)); + } + + /** + * Verifies all children still receive the callback when {@link ApiTracer.Scope#close()} throws. + */ + @ParameterizedTest(name = "{0}") + @MethodSource("scopedCallbacks") + void callbackReachesAllChildren_whenClosingScopeThrows( + String name, Consumer callback) { + ApiTracer.Scope brokenScope = mock(ApiTracer.Scope.class, withSettings().withoutAnnotations()); + doThrow(new IllegalStateException("broken close()")).when(brokenScope).close(); + when(child1.inScope()).thenReturn(brokenScope); + + callback.accept(compositeTracer); // must not throw + + callback.accept(verify(child1)); + callback.accept(verify(child2)); + verify(brokenScope).close(); + } +}