From 34893069a0676754cac53c5fc9562b288f1d9baa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Knut=20Olav=20L=C3=B8ite?= Date: Sat, 21 Nov 2026 15:51:33 +0100 Subject: [PATCH] test(spanner): add async mock server test infrastructure and tests Only modifies test suites and internal test-only helpers; no production runtime code is changed. Summary of changes: - Extend the async mock server test base class with client/database setup and automatic database/transport cleanup so async tests do not leak background tasks or channels across test cases. - Update the test-only async database and test interceptors to support multi-interface gRPC async interceptors, bytes header metadata, and async transaction abort/rollback simulation. - Add async mock server integration tests covering basic queries/streaming, aborted transaction retries, session pool binding, x-goog-spanner-request-id headers, and operation metrics, reusing shared assertion helpers across sync and async suites. - Replace wall-clock sleeps in the operation metrics tests with a deterministic fake clock. --- .../_async/testing/database_test.py | 65 +- .../spanner_v1/_async/testing/interceptors.py | 111 ++- .../cloud/spanner_v1/testing/database_test.py | 17 +- .../mockserver_tests/mock_server_test_base.py | 315 ++++--- .../test_aborted_transaction.py | 173 ++++ .../tests/mockserver_tests/test_basics.py | 242 ++++-- .../test_operation_metrics.py | 771 ++++++++++++------ .../mockserver_tests/test_pool_binding.py | 275 ++++--- .../test_request_id_header.py | 469 +++++++---- 9 files changed, 1688 insertions(+), 750 deletions(-) diff --git a/packages/google-cloud-spanner/google/cloud/spanner_v1/_async/testing/database_test.py b/packages/google-cloud-spanner/google/cloud/spanner_v1/_async/testing/database_test.py index d2c433eba4c9..25183bdcfa7b 100644 --- a/packages/google-cloud-spanner/google/cloud/spanner_v1/_async/testing/database_test.py +++ b/packages/google-cloud-spanner/google/cloud/spanner_v1/_async/testing/database_test.py @@ -20,10 +20,7 @@ from google.cloud.spanner_admin_database_v1 import DatabaseDialect from google.cloud.spanner_v1._async.database import Database from google.cloud.spanner_v1.database import SPANNER_DATA_SCOPE -from google.cloud.spanner_v1.services.spanner.transports import ( - SpannerGrpcTransport, - SpannerTransport, -) +from google.cloud.spanner_v1.services.spanner.transports import SpannerTransport if CrossSync.is_async: from google.cloud.spanner_v1._async.testing.interceptors import ( @@ -38,8 +35,14 @@ from google.cloud.spanner_v1.services.spanner.async_client import ( SpannerAsyncClient as SpannerClient, ) + from google.cloud.spanner_v1.services.spanner.transports.grpc_asyncio import ( + SpannerGrpcAsyncIOTransport as SpannerGrpcTransport, + ) else: from google.cloud.spanner_v1.services.spanner.client import SpannerClient + from google.cloud.spanner_v1.services.spanner.transports.grpc import ( + SpannerGrpcTransport, + ) from google.cloud.spanner_v1.testing.interceptors import ( MethodAbortInterceptor, MethodCountInterceptor, @@ -49,6 +52,28 @@ __CROSS_SYNC_OUTPUT__ = "google.cloud.spanner_v1.testing.database_test" +if CrossSync.is_async: + + def _attach_async_interceptors(channel, interceptors): + """Attach multi-interface async interceptors to all matching gRPC channel lists. + + grpc.aio.Channel.__init__ uses an if/elif chain on interceptor base classes, + so an interceptor implementing both UnaryUnaryClientInterceptor and + UnaryStreamClientInterceptor would otherwise only be attached to unary-unary. + """ + for interceptor in interceptors: + if isinstance(interceptor, grpc.aio.UnaryUnaryClientInterceptor): + channel._unary_unary_interceptors.append(interceptor) + if isinstance(interceptor, grpc.aio.UnaryStreamClientInterceptor): + channel._unary_stream_interceptors.append(interceptor) + if isinstance(interceptor, grpc.aio.StreamUnaryClientInterceptor): + channel._stream_unary_interceptors.append(interceptor) + if isinstance(interceptor, grpc.aio.StreamStreamClientInterceptor): + channel._stream_stream_interceptors.append(interceptor) + + +# TODO: Remove TestDatabase and enable_interceptors_in_tests after migrating +# mockserver, system, and unit tests to direct test transports. class TestDatabase(Database): """Representation of a Cloud Spanner Database. This class is only used for system testing as there is no support for interceptors in grpc client @@ -68,6 +93,7 @@ def __init__( database_dialect=DatabaseDialect.DATABASE_DIALECT_UNSPECIFIED, database_role=None, enable_drop_protection=False, + proto_descriptors=None, ): super().__init__( database_id, @@ -79,6 +105,7 @@ def __init__( database_dialect, database_role, enable_drop_protection, + proto_descriptors, ) self._method_count_interceptor = MethodCountInterceptor() @@ -95,21 +122,19 @@ def spanner_api(self): client = self._instance._client client_info = client._client_info client_options = client._client_options + if ( + not hasattr(self, "_x_goog_request_id_interceptor") + or self._x_goog_request_id_interceptor is None + ): + self._x_goog_request_id_interceptor = XGoogRequestIDHeaderInterceptor() + if self._x_goog_request_id_interceptor not in self._interceptors: + self._interceptors.append(self._x_goog_request_id_interceptor) if self._instance.emulator_host is not None: if CrossSync.is_async: - self._x_goog_request_id_interceptor = ( - XGoogRequestIDHeaderInterceptor() - ) - self._interceptors.append(self._x_goog_request_id_interceptor) - channel = grpc.aio.insecure_channel( - self._instance.emulator_host, interceptors=self._interceptors - ) + channel = grpc.aio.insecure_channel(self._instance.emulator_host) + _attach_async_interceptors(channel, self._interceptors) else: channel = grpc.insecure_channel(self._instance.emulator_host) - self._x_goog_request_id_interceptor = ( - XGoogRequestIDHeaderInterceptor() - ) - self._interceptors.append(self._x_goog_request_id_interceptor) channel = grpc.intercept_channel(channel, *self._interceptors) transport = SpannerGrpcTransport(channel=channel) @@ -119,9 +144,6 @@ def spanner_api(self): ) return self._spanner_api if getattr(client, "instance_type", None) == "omni": - self._x_goog_request_id_interceptor = XGoogRequestIDHeaderInterceptor() - self._interceptors.append(self._x_goog_request_id_interceptor) - from google.cloud.spanner_v1._async._helpers import ( _create_spanner_omni_transport as _create_spanner_omni_transport_async, ) @@ -137,9 +159,12 @@ def spanner_api(self): client._ca_certificate, client._client_certificate, client._client_key, - self._interceptors, + None, credentials=client.credentials, ) + _attach_async_interceptors( + transport.grpc_channel, self._interceptors + ) else: transport = _create_spanner_omni_transport_sync( SpannerGrpcTransport, @@ -181,8 +206,8 @@ def _create_spanner_client_for_tests(self, client_options, credentials): default_scopes=SpannerTransport.AUTH_SCOPES, scopes=client_options.scopes, default_host=SpannerTransport.DEFAULT_HOST, - interceptors=self._interceptors, ) + _attach_async_interceptors(channel, self._interceptors) else: channel = grpc_helpers.create_channel( api_endpoint, diff --git a/packages/google-cloud-spanner/google/cloud/spanner_v1/_async/testing/interceptors.py b/packages/google-cloud-spanner/google/cloud/spanner_v1/_async/testing/interceptors.py index ff279e4227f3..dcd8bd42a053 100644 --- a/packages/google-cloud-spanner/google/cloud/spanner_v1/_async/testing/interceptors.py +++ b/packages/google-cloud-spanner/google/cloud/spanner_v1/_async/testing/interceptors.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import inspect import threading from collections import defaultdict @@ -20,37 +21,83 @@ from google.cloud.spanner_v1.request_id_header import parse_request_id +X_GOOG_REQUEST_ID = "x-goog-spanner-request-id" + + +def _normalize_method_name(method) -> str: + """Decode gRPC method names from bytes to str when needed.""" + if isinstance(method, bytes): + return method.decode("utf-8") + return method + + +def _extract_request_id(client_call_details) -> str: + """Extract and validate the x-goog-spanner-request-id metadata header.""" + metadata = client_call_details.metadata or () + x_goog_request_id = None + for key, value in metadata: + decoded_key = key.decode("utf-8") if isinstance(key, bytes) else key + if decoded_key == X_GOOG_REQUEST_ID: + x_goog_request_id = ( + value.decode("utf-8") if isinstance(value, bytes) else value + ) + break + + if not x_goog_request_id: + method = _normalize_method_name(client_call_details.method) + raise ValueError(f"Missing {X_GOOG_REQUEST_ID} header in {method}") -class MethodCountAsyncInterceptor(grpc.aio.UnaryUnaryClientInterceptor): + return x_goog_request_id + + +class MethodCountAsyncInterceptor( + grpc.aio.UnaryUnaryClientInterceptor, + grpc.aio.UnaryStreamClientInterceptor, +): def __init__(self): self._counts = defaultdict(int) async def intercept_unary_unary(self, continuation, client_call_details, request): - self._counts[client_call_details.method] += 1 + self._counts[_normalize_method_name(client_call_details.method)] += 1 + return await continuation(client_call_details, request) + + async def intercept_unary_stream(self, continuation, client_call_details, request): + self._counts[_normalize_method_name(client_call_details.method)] += 1 return await continuation(client_call_details, request) def reset(self): self._counts = defaultdict(int) -class MethodAbortAsyncInterceptor(grpc.aio.UnaryUnaryClientInterceptor): +class MethodAbortAsyncInterceptor( + grpc.aio.UnaryUnaryClientInterceptor, + grpc.aio.UnaryStreamClientInterceptor, +): def __init__(self): self._method_to_abort = None self._count = 0 self._max_raise_count = 1 self._connection = None - async def intercept_unary_unary(self, continuation, client_call_details, request): - if ( - self._count < self._max_raise_count - and client_call_details.method == self._method_to_abort - ): + async def _maybe_raise_aborted(self, client_call_details): + method = _normalize_method_name(client_call_details.method) + if self._count < self._max_raise_count and method == self._method_to_abort: self._count += 1 - if self._connection is not None: - # Note: This assumes the connection rollback is sync or handled elsewhere - # For async connection, we might need a different approach if rollback is async - self._connection._transaction.rollback() + if ( + self._connection is not None + and self._connection._transaction is not None + ): + rollback_result = self._connection._transaction.rollback() + if inspect.isawaitable(rollback_result): + await rollback_result raise Aborted("Thrown from Async ClientInterceptor for testing") + + async def intercept_unary_unary(self, continuation, client_call_details, request): + await self._maybe_raise_aborted(client_call_details) + return await continuation(client_call_details, request) + + async def intercept_unary_stream(self, continuation, client_call_details, request): + await self._maybe_raise_aborted(client_call_details) return await continuation(client_call_details, request) def set_method_to_abort(self, method_to_abort, connection=None, max_raise_count=1): @@ -62,36 +109,31 @@ def set_method_to_abort(self, method_to_abort, connection=None, max_raise_count= def reset(self): self._method_to_abort = None self._count = 0 + self._max_raise_count = 1 self._connection = None -X_GOOG_REQUEST_ID = "x-goog-spanner-request-id" - - -class XGoogRequestIDHeaderAsyncInterceptor(grpc.aio.UnaryUnaryClientInterceptor): +class XGoogRequestIDHeaderAsyncInterceptor( + grpc.aio.UnaryUnaryClientInterceptor, + grpc.aio.UnaryStreamClientInterceptor, +): def __init__(self): self._unary_req_segments = [] self._stream_req_segments = [] self.__lock = threading.Lock() - async def intercept_unary_unary(self, continuation, client_call_details, request): - metadata = client_call_details.metadata - x_goog_request_id = None - for key, value in metadata: - if key == X_GOOG_REQUEST_ID: - x_goog_request_id = value - break - - if not x_goog_request_id: - raise Exception( - f"Missing {X_GOOG_REQUEST_ID} header in {client_call_details.method}" - ) - + def _record_request_id(self, client_call_details, target_segments): + x_goog_request_id = _extract_request_id(client_call_details) + method = _normalize_method_name(client_call_details.method) with self.__lock: - self._unary_req_segments.append( - (client_call_details.method, parse_request_id(x_goog_request_id)) - ) + target_segments.append((method, parse_request_id(x_goog_request_id))) + async def intercept_unary_unary(self, continuation, client_call_details, request): + self._record_request_id(client_call_details, self._unary_req_segments) + return await continuation(client_call_details, request) + + async def intercept_unary_stream(self, continuation, client_call_details, request): + self._record_request_id(client_call_details, self._stream_req_segments) return await continuation(client_call_details, request) @property @@ -103,5 +145,6 @@ def stream_request_ids(self): return self._stream_req_segments def reset(self): - self._stream_req_segments.clear() - self._unary_req_segments.clear() + with self.__lock: + self._stream_req_segments.clear() + self._unary_req_segments.clear() diff --git a/packages/google-cloud-spanner/google/cloud/spanner_v1/testing/database_test.py b/packages/google-cloud-spanner/google/cloud/spanner_v1/testing/database_test.py index 6018fcdeecdd..f702690dd684 100644 --- a/packages/google-cloud-spanner/google/cloud/spanner_v1/testing/database_test.py +++ b/packages/google-cloud-spanner/google/cloud/spanner_v1/testing/database_test.py @@ -22,9 +22,9 @@ from google.cloud.spanner_admin_database_v1 import DatabaseDialect from google.cloud.spanner_v1.database import SPANNER_DATA_SCOPE, Database from google.cloud.spanner_v1.services.spanner.client import SpannerClient -from google.cloud.spanner_v1.services.spanner.transports import ( +from google.cloud.spanner_v1.services.spanner.transports import SpannerTransport +from google.cloud.spanner_v1.services.spanner.transports.grpc import ( SpannerGrpcTransport, - SpannerTransport, ) from google.cloud.spanner_v1.testing.interceptors import ( MethodAbortInterceptor, @@ -52,6 +52,7 @@ def __init__( database_dialect=DatabaseDialect.DATABASE_DIALECT_UNSPECIFIED, database_role=None, enable_drop_protection=False, + proto_descriptors=None, ): super().__init__( database_id, @@ -63,6 +64,7 @@ def __init__( database_dialect, database_role, enable_drop_protection, + proto_descriptors, ) self._method_count_interceptor = MethodCountInterceptor() self._method_abort_interceptor = MethodAbortInterceptor() @@ -78,10 +80,15 @@ def spanner_api(self): client = self._instance._client client_info = client._client_info client_options = client._client_options - if self._instance.emulator_host is not None: - channel = grpc.insecure_channel(self._instance.emulator_host) + if ( + not hasattr(self, "_x_goog_request_id_interceptor") + or self._x_goog_request_id_interceptor is None + ): self._x_goog_request_id_interceptor = XGoogRequestIDHeaderInterceptor() + if self._x_goog_request_id_interceptor not in self._interceptors: self._interceptors.append(self._x_goog_request_id_interceptor) + if self._instance.emulator_host is not None: + channel = grpc.insecure_channel(self._instance.emulator_host) channel = grpc.intercept_channel(channel, *self._interceptors) transport = SpannerGrpcTransport(channel=channel) self._spanner_api = SpannerClient( @@ -89,8 +96,6 @@ def spanner_api(self): ) return self._spanner_api if getattr(client, "instance_type", None) == "omni": - self._x_goog_request_id_interceptor = XGoogRequestIDHeaderInterceptor() - self._interceptors.append(self._x_goog_request_id_interceptor) from google.cloud.spanner_v1._helpers import ( _create_spanner_omni_transport as _create_spanner_omni_transport_sync, ) diff --git a/packages/google-cloud-spanner/tests/mockserver_tests/mock_server_test_base.py b/packages/google-cloud-spanner/tests/mockserver_tests/mock_server_test_base.py index 525f1bd434c0..9067a56685b5 100644 --- a/packages/google-cloud-spanner/tests/mockserver_tests/mock_server_test_base.py +++ b/packages/google-cloud-spanner/tests/mockserver_tests/mock_server_test_base.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import inspect import logging import os import unittest @@ -224,6 +225,8 @@ def add_single_result( ) +# TODO: Construct test Database/transport directly instead of relying on +# enable_interceptors_in_tests=True. class MockServerTestBase(unittest.TestCase): _interceptors = [] server: grpc.Server = None @@ -306,58 +309,13 @@ def assert_requests_sequence( transaction_type, allow_multiple_batch_create=True, ): - """Assert that the requests sequence matches the expected types, accounting for multiplexed sessions and retries. - - Args: - requests: List of requests from spanner_service.requests - expected_types: List of expected request types (excluding session creation requests) - transaction_type: TransactionType enum value to check multiplexed session status - allow_multiple_batch_create: If True, skip all leading BatchCreateSessionsRequest and one optional CreateSessionRequest - """ - mux_enabled = is_multiplexed_enabled(transaction_type) - idx = 0 - # Skip all leading BatchCreateSessionsRequest (for retries) - if allow_multiple_batch_create: - while ( - idx < len(requests) - and type(requests[idx]).__name__ == "BatchCreateSessionsRequest" - ): - idx += 1 - # For multiplexed, optionally skip a CreateSessionRequest - if ( - mux_enabled - and idx < len(requests) - and type(requests[idx]).__name__ == "CreateSessionRequest" - ): - idx += 1 - else: - if mux_enabled: - self.assertTrue( - type(requests[idx]).__name__ == "BatchCreateSessionsRequest", - f"Expected BatchCreateSessionsRequest at index {idx}, got {type(requests[idx])}", - ) - idx += 1 - self.assertTrue( - type(requests[idx]).__name__ == "CreateSessionRequest", - f"Expected CreateSessionRequest at index {idx}, got {type(requests[idx])}", - ) - idx += 1 - else: - self.assertTrue( - type(requests[idx]).__name__ == "BatchCreateSessionsRequest", - f"Expected BatchCreateSessionsRequest at index {idx}, got {type(requests[idx])}", - ) - idx += 1 - # Check the rest of the expected request types - for expected_type in expected_types: - self.assertTrue( - isinstance(requests[idx], expected_type) - or type(requests[idx]).__name__ == expected_type.__name__, - f"Expected {expected_type} at index {idx}, got {type(requests[idx])}", - ) - idx += 1 - self.assertEqual( - idx, len(requests), f"Expected {idx} requests, got {len(requests)}" + """Assert that the requests sequence matches expected types.""" + _assert_requests_sequence( + self, + requests, + expected_types, + transaction_type, + allow_multiple_batch_create=allow_multiple_batch_create, ) def adjust_request_id_sequence(self, expected_segments, requests, transaction_type): @@ -399,6 +357,67 @@ def adjust_request_id_sequence(self, expected_segments, requests, transaction_ty return adjusted_segments +def _assert_requests_sequence( + test_case, + requests, + expected_types, + transaction_type, + allow_multiple_batch_create=True, +): + """Assert that requests match expected types, accounting for multiplexed sessions and retries.""" + mux_enabled = is_multiplexed_enabled(transaction_type) + request_index = 0 + # Skip all leading BatchCreateSessionsRequest (for retries) + if allow_multiple_batch_create: + while ( + request_index < len(requests) + and type(requests[request_index]).__name__ == "BatchCreateSessionsRequest" + ): + request_index += 1 + # For multiplexed, optionally skip a CreateSessionRequest + if ( + mux_enabled + and request_index < len(requests) + and type(requests[request_index]).__name__ == "CreateSessionRequest" + ): + request_index += 1 + else: + expected_leading = ["BatchCreateSessionsRequest"] + if mux_enabled: + expected_leading.append("CreateSessionRequest") + for expected_name in expected_leading: + test_case.assertLess( + request_index, + len(requests), + f"Expected {expected_name} at index {request_index}, got {requests}", + ) + test_case.assertTrue( + type(requests[request_index]).__name__ == expected_name, + f"Expected {expected_name} at index {request_index}, got {type(requests[request_index])}", + ) + request_index += 1 + # Check the rest of the expected request types + for expected_type in expected_types: + test_case.assertLess( + request_index, + len(requests), + f"Expected {expected_type} at index {request_index}, got only {len(requests)} requests", + ) + test_case.assertTrue( + isinstance(requests[request_index], expected_type) + or type(requests[request_index]).__name__ == expected_type.__name__, + f"Expected {expected_type} at index {request_index}, got {type(requests[request_index])}", + ) + request_index += 1 + test_case.assertEqual( + request_index, + len(requests), + f"Expected {request_index} requests, got {len(requests)}", + ) + + +# TODO: Construct test Database/transport directly instead of relying on +# enable_interceptors_in_tests=True. class AsyncMockServerTestBase(unittest.IsolatedAsyncioTestCase): server: grpc.Server = None spanner_service: SpannerServicer = None @@ -411,6 +430,7 @@ def __init__(self, *args, **kwargs): self._client = None self._instance = None self._database = None + self._databases_to_close = [] self.logger = logging.getLogger("AsyncMockServerTestBase") self.logger.setLevel(logging.WARN) @@ -427,35 +447,75 @@ def setUpClass(cls): def tearDownClass(cls): if AsyncMockServerTestBase.server is not None: AsyncMockServerTestBase.server.stop(grace=None) - from google.cloud.spanner_v1.client import Client as SyncClient + from google.cloud.spanner_v1._async.client import Client as AsyncClient - SyncClient.NTH_CLIENT.reset() + AsyncClient.NTH_CLIENT.reset() AsyncMockServerTestBase.server = None async def asyncSetUp(self): self._client = None self._instance = None self._database = None + self._databases_to_close = [] current_service.set(AsyncMockServerTestBase.spanner_service) + async def _close_database_and_transport(self, database): + """Close a database's background session tasks and its gRPC AsyncIO transport.""" + try: + await database.close() + finally: + spanner_api = getattr(database, "_spanner_api", None) + transport = getattr(spanner_api, "transport", None) if spanner_api else None + if transport is not None and hasattr(transport, "close"): + close_result = transport.close() + if inspect.isawaitable(close_result): + await close_result + async def asyncTearDown(self): - AsyncMockServerTestBase.spanner_service.clear_requests() - AsyncMockServerTestBase.spanner_service.clear_results() - AsyncMockServerTestBase.database_admin_service.clear_requests() + first_error = None + try: + for database in reversed(self._databases_to_close): + try: + await self._close_database_and_transport(database) + except Exception as error: + if first_error is None: + first_error = error + self._databases_to_close.clear() + finally: + AsyncMockServerTestBase.spanner_service.clear_requests() + AsyncMockServerTestBase.spanner_service.clear_results() + AsyncMockServerTestBase.database_admin_service.clear_requests() + if first_error is not None: + raise first_error + + def create_client( + self, + *, + disable_builtin_metrics: bool = True, + observability_options: dict | None = None, + client_options: ClientOptions | None = None, + **kwargs, + ): + """Create an AsyncClient configured against the active mock server.""" + from google.cloud.spanner_v1._async.client import Client as AsyncClient + + if client_options is None: + client_options = ClientOptions( + api_endpoint=f"localhost:{AsyncMockServerTestBase.port}", + ) + return AsyncClient( + project="p", + credentials=AnonymousCredentials(), + client_options=client_options, + disable_builtin_metrics=disable_builtin_metrics, + observability_options=observability_options, + **kwargs, + ) @property def client(self): - from google.cloud.spanner_v1._async.client import Client as AsyncClient - if self._client is None: - self._client = AsyncClient( - project="p", - credentials=AnonymousCredentials(), - client_options=ClientOptions( - api_endpoint="localhost:" + str(AsyncMockServerTestBase.port), - ), - disable_builtin_metrics=True, - ) + self._client = self.create_client() return self._client @property @@ -464,17 +524,63 @@ def instance(self): self._instance = self.client.instance("test-instance") return self._instance - @property - def database(self): + async def create_database( + self, + database_id: str = "test-database", + *, + client=None, + instance=None, + pool=None, + enable_interceptors_in_tests: bool = False, + interceptors=None, + logger: logging.Logger | None = None, + **kwargs, + ): + """Create, bind, and track an async Database for automatic teardown.""" from google.cloud.spanner_v1._async.pool import FixedSizePool - if self._database is None: - self._database = self.instance.database( - "test-database", - pool=FixedSizePool(size=10), - enable_interceptors_in_tests=False, - logger=self.logger, + if interceptors is not None and not enable_interceptors_in_tests: + raise ValueError( + "enable_interceptors_in_tests must be True when interceptors are provided." + ) + + target_instance = instance or ( + client.instance("test-instance") if client is not None else self.instance + ) + if pool is None: + pool = FixedSizePool(size=10) + + if interceptors is not None: + from google.cloud.spanner_v1._async.testing.database_test import ( + TestDatabase, + ) + + database = TestDatabase( + database_id, + target_instance, + pool=pool, + logger=logger or self.logger, + **kwargs, ) + database._interceptors.extend(interceptors) + bind_result = database._pool.bind(database) + if bind_result is not None: + await bind_result + else: + database = await target_instance.database( + database_id, + pool=pool, + enable_interceptors_in_tests=enable_interceptors_in_tests, + logger=logger or self.logger, + **kwargs, + ) + self._databases_to_close.append(database) + return database + + async def get_database(self): + """Return a lazily-created default async Database bound to the mock server.""" + if self._database is None: + self._database = await self.create_database() return self._database def assert_requests_sequence( @@ -484,56 +590,11 @@ def assert_requests_sequence( transaction_type, allow_multiple_batch_create=True, ): - """Assert that the requests sequence matches the expected types, accounting for multiplexed sessions and retries. - - Args: - requests: List of requests from spanner_service.requests - expected_types: List of expected request types (excluding session creation requests) - transaction_type: TransactionType enum value to check multiplexed session status - allow_multiple_batch_create: If True, skip all leading BatchCreateSessionsRequest and one optional CreateSessionRequest - """ - mux_enabled = is_multiplexed_enabled(transaction_type) - idx = 0 - # Skip all leading BatchCreateSessionsRequest (for retries) - if allow_multiple_batch_create: - while ( - idx < len(requests) - and type(requests[idx]).__name__ == "BatchCreateSessionsRequest" - ): - idx += 1 - # For multiplexed, optionally skip a CreateSessionRequest - if ( - mux_enabled - and idx < len(requests) - and type(requests[idx]).__name__ == "CreateSessionRequest" - ): - idx += 1 - else: - if mux_enabled: - self.assertTrue( - type(requests[idx]).__name__ == "BatchCreateSessionsRequest", - f"Expected BatchCreateSessionsRequest at index {idx}, got {type(requests[idx])}", - ) - idx += 1 - self.assertTrue( - type(requests[idx]).__name__ == "CreateSessionRequest", - f"Expected CreateSessionRequest at index {idx}, got {type(requests[idx])}", - ) - idx += 1 - else: - self.assertTrue( - type(requests[idx]).__name__ == "BatchCreateSessionsRequest", - f"Expected BatchCreateSessionsRequest at index {idx}, got {type(requests[idx])}", - ) - idx += 1 - # Check the rest of the expected request types - for expected_type in expected_types: - self.assertTrue( - isinstance(requests[idx], expected_type) - or type(requests[idx]).__name__ == expected_type.__name__, - f"Expected {expected_type} at index {idx}, got {type(requests[idx])}", - ) - idx += 1 - self.assertEqual( - idx, len(requests), f"Expected {idx} requests, got {len(requests)}" + """Assert that the requests sequence matches expected types.""" + _assert_requests_sequence( + self, + requests, + expected_types, + transaction_type, + allow_multiple_batch_create=allow_multiple_batch_create, ) diff --git a/packages/google-cloud-spanner/tests/mockserver_tests/test_aborted_transaction.py b/packages/google-cloud-spanner/tests/mockserver_tests/test_aborted_transaction.py index dc21b36d90ea..493450d3a000 100644 --- a/packages/google-cloud-spanner/tests/mockserver_tests/test_aborted_transaction.py +++ b/packages/google-cloud-spanner/tests/mockserver_tests/test_aborted_transaction.py @@ -23,10 +23,12 @@ ReadRequest, TypeCode, ) +from google.cloud.spanner_v1._async.transaction import Transaction as AsyncTransaction from google.cloud.spanner_v1.database_sessions_manager import TransactionType from google.cloud.spanner_v1.testing.mock_spanner import SpannerServicer from google.cloud.spanner_v1.transaction import Transaction from tests.mockserver_tests.mock_server_test_base import ( + AsyncMockServerTestBase, MockServerTestBase, aborted_status, add_error, @@ -218,6 +220,154 @@ def do_commit(): do_commit() +class TestAbortedTransactionAsync(AsyncMockServerTestBase): + async def test_async_run_in_transaction_commit_aborted(self): + add_error(SpannerServicer.Commit.__name__, aborted_status()) + database = await self.get_database() + await database.run_in_transaction(_async_insert_mutations) + requests = self.spanner_service.requests + self.assert_requests_sequence( + requests, + [ + BeginTransactionRequest, + CommitRequest, + BeginTransactionRequest, + CommitRequest, + ], + TransactionType.READ_WRITE, + ) + + async def test_async_run_in_transaction_update_aborted(self): + add_update_count("update my_table set my_col=1 where id=2", 1) + add_error(SpannerServicer.ExecuteSql.__name__, aborted_status()) + database = await self.get_database() + await database.run_in_transaction(_async_execute_update) + requests = self.spanner_service.requests + self.assert_requests_sequence( + requests, + [ExecuteSqlRequest, ExecuteSqlRequest, CommitRequest], + TransactionType.READ_WRITE, + ) + + async def test_async_run_in_transaction_query_aborted(self): + add_single_result( + "select value from my_table where id=1", + "value", + TypeCode.STRING, + "my-value", + ) + add_error(SpannerServicer.ExecuteStreamingSql.__name__, aborted_status()) + database = await self.get_database() + await database.run_in_transaction(_async_execute_query) + requests = self.spanner_service.requests + self.assert_requests_sequence( + requests, + [ExecuteSqlRequest, ExecuteSqlRequest, CommitRequest], + TransactionType.READ_WRITE, + ) + + async def test_async_run_in_transaction_query_mid_stream_aborted(self): + sql = "select value from my_table where id=1" + add_single_result( + sql, + "value", + TypeCode.STRING, + [("row1",), ("row2",)], + ) + add_execute_streaming_sql_error(sql, 1, aborted_status()) + + observed_attempts = [] + partial_rows_seen = [] + + async def _query_tracking_partials(transaction: AsyncTransaction): + attempt_rows = [] + async for row in await transaction.execute_sql(sql): + attempt_rows.append(row) + partial_rows_seen.append(row) + observed_attempts.append(attempt_rows) + return attempt_rows + + database = await self.get_database() + result = await database.run_in_transaction(_query_tracking_partials) + self.assertEqual([["row1"], ["row2"]], result) + self.assertEqual([[["row1"], ["row2"]]], observed_attempts) + self.assertEqual([["row1"], ["row1"], ["row2"]], partial_rows_seen) + requests = self.spanner_service.requests + self.assert_requests_sequence( + requests, + [ExecuteSqlRequest, ExecuteSqlRequest, CommitRequest], + TransactionType.READ_WRITE, + ) + + async def test_async_run_in_transaction_streaming_read_mid_stream_aborted(self): + add_single_result( + "SELECT value FROM my_table", + "value", + TypeCode.STRING, + [("row1",), ("row2",)], + ) + add_streaming_read_error("my_table", ["value"], 1, aborted_status()) + + partial_rows_seen = [] + + async def _read_tracking_partials(transaction: AsyncTransaction): + attempt_rows = [] + async for row in await transaction.read( + table="my_table", + columns=["value"], + keyset=KeySet(all_=True), + ): + attempt_rows.append(row) + partial_rows_seen.append(row) + return attempt_rows + + database = await self.get_database() + result = await database.run_in_transaction(_read_tracking_partials) + self.assertEqual([["row1"], ["row2"]], result) + self.assertEqual([["row1"], ["row1"], ["row2"]], partial_rows_seen) + requests = self.spanner_service.requests + self.assert_requests_sequence( + requests, + [ReadRequest, ReadRequest, CommitRequest], + TransactionType.READ_WRITE, + ) + + async def test_async_run_in_transaction_batch_dml_aborted(self): + add_update_count("update my_table set my_col=1 where id=1", 1) + add_update_count("update my_table set my_col=1 where id=2", 1) + add_error(SpannerServicer.ExecuteBatchDml.__name__, aborted_status()) + database = await self.get_database() + await database.run_in_transaction(_async_execute_batch_dml) + requests = self.spanner_service.requests + self.assert_requests_sequence( + requests, + [ExecuteBatchDmlRequest, ExecuteBatchDmlRequest, CommitRequest], + TransactionType.READ_WRITE, + ) + + async def test_async_batch_commit_aborted(self): + add_error(SpannerServicer.Commit.__name__, aborted_status()) + database = await self.get_database() + async with database.batch() as batch: + batch.insert( + table="Singers", + columns=("SingerId", "FirstName", "LastName"), + values=[ + (1, "Marc", "Richards"), + (2, "Catalina", "Smith"), + (3, "Alice", "Trentor"), + (4, "Lea", "Martin"), + (5, "David", "Lomond"), + ], + ) + requests = self.spanner_service.requests + self.assert_requests_sequence( + requests, + [CommitRequest, CommitRequest], + TransactionType.READ_WRITE, + ) + + def _insert_mutations(transaction: Transaction): transaction.insert("my_table", ["col1", "col2"], ["value1", "value2"]) @@ -239,3 +389,26 @@ def _execute_batch_dml(transaction: Transaction): "update my_table set my_col=1 where id=2", ] ) + + +async def _async_insert_mutations(transaction: AsyncTransaction): + transaction.insert("my_table", ["col1", "col2"], ["value1", "value2"]) + + +async def _async_execute_update(transaction: AsyncTransaction): + await transaction.execute_update("update my_table set my_col=1 where id=2") + + +async def _async_execute_query(transaction: AsyncTransaction): + rows = await transaction.execute_sql("select value from my_table where id=1") + async for _ in rows: + pass + + +async def _async_execute_batch_dml(transaction: AsyncTransaction): + await transaction.batch_update( + [ + "update my_table set my_col=1 where id=1", + "update my_table set my_col=1 where id=2", + ] + ) diff --git a/packages/google-cloud-spanner/tests/mockserver_tests/test_basics.py b/packages/google-cloud-spanner/tests/mockserver_tests/test_basics.py index 1eae77319b43..064847ba9c8a 100644 --- a/packages/google-cloud-spanner/tests/mockserver_tests/test_basics.py +++ b/packages/google-cloud-spanner/tests/mockserver_tests/test_basics.py @@ -13,12 +13,8 @@ # limitations under the License. import unittest -from contextlib import asynccontextmanager -import grpc.aio from google.api_core import exceptions -from google.api_core.client_options import ClientOptions -from google.auth.credentials import AnonymousCredentials from google.cloud.spanner_admin_database_v1.types import spanner_database_admin from google.cloud.spanner_dbapi import Connection @@ -33,13 +29,8 @@ TransactionSelector, TypeCode, ) -from google.cloud.spanner_v1._async.client import Client as AsyncClient -from google.cloud.spanner_v1._async.pool import FixedSizePool as AsyncFixedSizePool +from google.cloud.spanner_v1._async.transaction import Transaction as AsyncTransaction from google.cloud.spanner_v1.database_sessions_manager import TransactionType -from google.cloud.spanner_v1.services.spanner.async_client import SpannerAsyncClient -from google.cloud.spanner_v1.services.spanner.transports import ( - SpannerGrpcAsyncIOTransport, -) from google.cloud.spanner_v1.testing.mock_spanner import ( PartialResultSetConfig, SpannerServicer, @@ -1073,28 +1064,22 @@ def test_dbapi_fetchall_mid_stream_aborted_issue_19079(self): class TestBasicsAsync(AsyncMockServerTestBase): - @asynccontextmanager - async def _async_database(self): - client = AsyncClient( - project="p", - credentials=AnonymousCredentials(), - client_options=ClientOptions( - api_endpoint=f"localhost:{AsyncMockServerTestBase.port}" - ), - disable_builtin_metrics=True, - ) - instance = client.instance("test-instance") - database = await instance.database( - "test-database", pool=AsyncFixedSizePool(size=10) - ) - async with grpc.aio.insecure_channel( - f"localhost:{AsyncMockServerTestBase.port}" - ) as channel: - database._spanner_api = SpannerAsyncClient( - client_info=client._client_info, - transport=SpannerGrpcAsyncIOTransport(channel=channel), - ) - yield database + async def test_async_select1(self): + add_select1_result() + database = await self.get_database() + async with database.snapshot() as snapshot: + results = await snapshot.execute_sql("select 1") + result_list = [] + async for row in results: + result_list.append(row) + self.assertEqual(1, row[0]) + self.assertEqual(1, len(result_list)) + requests = self.spanner_service.requests + self.assert_requests_sequence( + requests, + [ExecuteSqlRequest], + TransactionType.READ_ONLY, + ) async def test_async_execute_streaming_sql_restarts_without_resume_token(self): # Verify chunk 0 without a resume_token (chunked_value=True) restarts from b"". @@ -1108,11 +1093,11 @@ async def test_async_execute_streaming_sql_restarts_without_resume_token(self): ), ) add_execute_streaming_sql_error(sql, 1, unavailable_status()) - async with self._async_database() as database: - async with database.snapshot() as snapshot: - results = await snapshot.execute_sql(sql) - rows = [row async for row in results] - self.assertEqual([[1, "hello world"], [2, "abcdef"]], rows) + database = await self.get_database() + async with database.snapshot() as snapshot: + results = await snapshot.execute_sql(sql) + rows = [row async for row in results] + self.assertEqual([[1, "hello world"], [2, "abcdef"]], rows) execute_requests = _requests_of_type(self.spanner_service, ExecuteSqlRequest) self.assertEqual( [b"", b""], [request.resume_token for request in execute_requests] @@ -1128,11 +1113,11 @@ async def test_async_execute_streaming_sql_resumes_with_resume_token(self): chunk_config=PartialResultSetConfig(max_string_chunk_length=4), ) add_execute_streaming_sql_error(sql, 3, unavailable_status()) - async with self._async_database() as database: - async with database.snapshot() as snapshot: - results = await snapshot.execute_sql(sql) - rows = [row async for row in results] - self.assertEqual([[1, "hello world"], [2, "abcdef"]], rows) + database = await self.get_database() + async with database.snapshot() as snapshot: + results = await snapshot.execute_sql(sql) + rows = [row async for row in results] + self.assertEqual([[1, "hello world"], [2, "abcdef"]], rows) execute_requests = _requests_of_type(self.spanner_service, ExecuteSqlRequest) self.assertEqual( [b"", b"1"], [request.resume_token for request in execute_requests] @@ -1148,15 +1133,15 @@ async def test_async_streaming_read_resumes_with_resume_token(self): add_streaming_read_error( "async_read_docs", ["id", "payload"], 1, unavailable_status() ) - async with self._async_database() as database: - async with database.snapshot() as snapshot: - read_results = await snapshot.read( - table="async_read_docs", - columns=["id", "payload"], - keyset=KeySet(all_=True), - ) - read_rows = [row async for row in read_results] - self.assertEqual([[1, "hello world"], [2, "abcdef"]], read_rows) + database = await self.get_database() + async with database.snapshot() as snapshot: + read_results = await snapshot.read( + table="async_read_docs", + columns=["id", "payload"], + keyset=KeySet(all_=True), + ) + read_rows = [row async for row in read_results] + self.assertEqual([[1, "hello world"], [2, "abcdef"]], read_rows) read_requests = _requests_of_type(self.spanner_service, ReadRequest) self.assertEqual( [b"", b"1"], [request.resume_token for request in read_requests] @@ -1172,15 +1157,15 @@ async def test_async_execute_streaming_sql_mid_stream_deadline_exceeded(self): chunk_config=PartialResultSetConfig(max_string_chunk_length=4), ) add_execute_streaming_sql_error(sql, 3, deadline_exceeded_status()) - async with self._async_database() as database: - async with database.snapshot() as snapshot: - results = await snapshot.execute_sql(sql) - row_iterator = results.__aiter__() - self.assertEqual([1, "hello world"], await row_iterator.__anext__()) - with self.assertRaisesRegex( - exceptions.DeadlineExceeded, "Deadline exceeded." - ): - await row_iterator.__anext__() + database = await self.get_database() + async with database.snapshot() as snapshot: + results = await snapshot.execute_sql(sql) + row_iterator = aiter(results) + self.assertEqual([1, "hello world"], await anext(row_iterator)) + with self.assertRaisesRegex( + exceptions.DeadlineExceeded, "Deadline exceeded." + ): + await anext(row_iterator) async def test_async_execute_streaming_sql_last_flag_trailing_error_drained(self): # Verify async _drain_stream absorbs trailing errors sent after last=True. @@ -1193,11 +1178,138 @@ async def test_async_execute_streaming_sql_last_flag_trailing_error_drained(self chunk_config=PartialResultSetConfig(set_last_flag=True), ) add_execute_streaming_sql_error(sql, 5, internal_server_error_status()) - async with self._async_database() as database: - async with database.snapshot() as snapshot: - results = await snapshot.execute_sql(sql) - rows = [row async for row in results] - self.assertEqual([[10], [20]], rows) + database = await self.get_database() + async with database.snapshot() as snapshot: + results = await snapshot.execute_sql(sql) + rows = [row async for row in results] + self.assertEqual([[10], [20]], rows) + + async def test_async_execute_streaming_sql_multiple_resumptions_and_rst_stream( + self, + ): + sql = "select id from async_multi_resume" + add_single_result(sql, "id", TypeCode.INT64, [("1",), ("2",), ("3",), ("4",)]) + add_execute_streaming_sql_error(sql, 1, unavailable_status()) + add_execute_streaming_sql_error( + sql, + 1, + internal_server_error_status("Received RST_STREAM with error code 2"), + ) + database = await self.get_database() + async with database.snapshot() as snapshot: + results = await snapshot.execute_sql(sql) + self.assertEqual([[1], [2], [3], [4]], [row async for row in results]) + execute_requests = _requests_of_type(self.spanner_service, ExecuteSqlRequest) + self.assertEqual( + [b"", b"1", b"2"], + [request.resume_token for request in execute_requests], + ) + + async def test_async_inline_begin_transaction_mid_stream_unavailable_switches_to_id( + self, + ): + sql = "select id from async_rw_resume" + add_single_result(sql, "id", TypeCode.INT64, [("1",), ("2",)]) + add_execute_streaming_sql_error(sql, 1, unavailable_status()) + + async def _query(transaction: AsyncTransaction): + results = await transaction.execute_sql(sql) + return [row async for row in results] + + database = await self.get_database() + rows = await database.run_in_transaction(_query) + self.assertEqual([[1], [2]], rows) + execute_requests = _requests_of_type(self.spanner_service, ExecuteSqlRequest) + self.assertEqual(2, len(execute_requests)) + self.assertNotEqual(TransactionOptions(), execute_requests[0].transaction.begin) + self.assertEqual(b"", execute_requests[0].resume_token) + self.assertEqual(TransactionOptions(), execute_requests[1].transaction.begin) + self.assertTrue(execute_requests[1].transaction.id) + self.assertEqual(b"1", execute_requests[1].resume_token) + + async def test_async_streaming_read_chunking_and_boundary_errors(self): + add_multi_column_result( + "SELECT id, payload FROM async_chunked_read", + [("id", TypeCode.INT64), ("payload", TypeCode.STRING)], + [("1", "hello world"), ("2", "ok")], + chunk_config=PartialResultSetConfig( + max_values_per_chunk=2, + max_string_chunk_length=4, + ), + ) + add_streaming_read_error( + "async_chunked_read", ["id", "payload"], 1, unavailable_status() + ) + database = await self.get_database() + async with database.snapshot() as snapshot: + read_results = await snapshot.read( + table="async_chunked_read", + columns=["id", "payload"], + keyset=KeySet(all_=True), + ) + self.assertEqual( + [[1, "hello world"], [2, "ok"]], + [row async for row in read_results], + ) + + add_streaming_read_error( + "async_chunked_read", ["id", "payload"], 0, invalid_argument_status() + ) + async with database.snapshot() as snapshot: + read_results = await snapshot.read( + table="async_chunked_read", + columns=["id", "payload"], + keyset=KeySet(all_=True), + ) + row_iterator = aiter(read_results) + with self.assertRaisesRegex( + exceptions.InvalidArgument, "Invalid argument." + ): + await anext(row_iterator) + + add_streaming_read_error( + "async_chunked_read", ["id", "payload"], 10, internal_server_error_status() + ) + async with database.snapshot() as snapshot: + read_results = await snapshot.read( + table="async_chunked_read", + columns=["id", "payload"], + keyset=KeySet(all_=True), + ) + row_iterator = aiter(read_results) + self.assertEqual([1, "hello world"], await anext(row_iterator)) + self.assertEqual([2, "ok"], await anext(row_iterator)) + with self.assertRaisesRegex( + exceptions.InternalServerError, "Internal server error." + ): + await anext(row_iterator) + + async def test_async_streamed_result_set_helpers_with_chunked_streams(self): + sql_one = "select id, payload from async_helper_one" + add_multi_column_result( + sql_one, + [("id", TypeCode.INT64), ("payload", TypeCode.STRING)], + [("7", "hello world")], + chunk_config=PartialResultSetConfig( + max_values_per_chunk=1, max_string_chunk_length=4 + ), + ) + database = await self.get_database() + async with database.snapshot(multi_use=True) as snapshot: + self.assertEqual( + [7, "hello world"], await (await snapshot.execute_sql(sql_one)).one() + ) + self.assertEqual( + [7, "hello world"], + await (await snapshot.execute_sql(sql_one)).one_or_none(), + ) + self.assertEqual( + [{"id": 7, "payload": "hello world"}], + await (await snapshot.execute_sql(sql_one)).to_dict_list(), + ) + lazy_results = await snapshot.execute_sql(sql_one, lazy_decode=True) + lazy_rows = [row async for row in lazy_results] + self.assertEqual([7, "hello world"], lazy_results.decode_row(lazy_rows[0])) def _execute_query(transaction: Transaction, sql: str): diff --git a/packages/google-cloud-spanner/tests/mockserver_tests/test_operation_metrics.py b/packages/google-cloud-spanner/tests/mockserver_tests/test_operation_metrics.py index f9659e71a4af..e82f16b05bff 100644 --- a/packages/google-cloud-spanner/tests/mockserver_tests/test_operation_metrics.py +++ b/packages/google-cloud-spanner/tests/mockserver_tests/test_operation_metrics.py @@ -13,14 +13,18 @@ # limitations under the License. import os -import time +import threading +from contextlib import contextmanager +from datetime import datetime, timedelta from unittest import mock +from google.api_core import exceptions from google.api_core.client_options import ClientOptions from google.auth.credentials import AnonymousCredentials from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.metrics.export import InMemoryMetricReader +import google.cloud.spanner_v1._async.client as async_client_mod import google.cloud.spanner_v1.client as client_mod from google.cloud.spanner_v1 import ( Client, @@ -28,35 +32,193 @@ KeySet, TypeCode, ) -from google.cloud.spanner_v1.metrics.metrics_interceptor import MetricsInterceptor +from google.cloud.spanner_v1._async.pool import FixedSizePool as AsyncFixedSizePool +from google.cloud.spanner_v1.metrics.metrics_interceptor import ( + AsyncMetricsInterceptor, + MetricsInterceptor, +) from google.cloud.spanner_v1.metrics.spanner_metrics_tracer_factory import ( SpannerMetricsTracerFactory, ) -from google.cloud.spanner_v1.testing.mock_spanner import SpannerServicer +from google.cloud.spanner_v1.testing.mock_spanner import ( + PartialResultSetConfig, + SpannerServicer, +) from tests.mockserver_tests.mock_server_test_base import ( + AsyncMockServerTestBase, MockServerTestBase, _make_partial_result_sets, add_error, + add_execute_streaming_sql_error, add_execute_streaming_sql_results, + add_multi_column_result, add_select1_result, + add_single_result, + deadline_exceeded_status, unavailable_status, ) -class TestOperationMetricsIntegration(MockServerTestBase): +class _OperationMetricsAssertionsMixin: + @staticmethod + def extract_metrics(reader, method: str | None = None) -> dict: + """Extract metric data points from an InMemoryMetricReader, optionally filtered by RPC method.""" + metric_data = reader.get_metrics_data() + if not metric_data: + return {} + all_metrics = ( + metric + for resource_metric in metric_data.resource_metrics + for scope_metric in resource_metric.scope_metrics + for metric in scope_metric.metrics + ) + metrics = {} + for metric in all_metrics: + points = [ + point + for point in metric.data.data_points + if not method or point.attributes.get("method") == method + ] + if points: + metrics[metric.name] = points + return metrics + + @staticmethod + @contextmanager + def patched_metrics_environment(meter_provider): + """Patch OpenTelemetry MeterProvider and emulator host detection for sync and async clients.""" + with ( + mock.patch.dict( + os.environ, + {"SPANNER_DISABLE_BUILTIN_METRICS": "false"}, + ), + mock.patch( + "google.cloud.spanner_v1.metrics.metrics_tracer_factory.get_meter_provider", + return_value=meter_provider, + ), + mock.patch( + "google.cloud.spanner_v1.client.MeterProvider", + return_value=meter_provider, + ), + mock.patch( + "google.cloud.spanner_v1._async.client.MeterProvider", + return_value=meter_provider, + ), + mock.patch( + "google.cloud.spanner_v1.client._get_spanner_emulator_host", + return_value=None, + ), + mock.patch( + "google.cloud.spanner_v1._async.client._get_spanner_emulator_host", + return_value=None, + ), + ): + yield + + @contextmanager + def _delayed_streaming_metrics(self, chunk_delay_milliseconds: int): + """Yield an InMemoryMetricReader with deterministic per-chunk clock advances.""" + SpannerMetricsTracerFactory._metrics_tracer_factory = None + client_mod._metrics_monitor_initialized = False + async_client_mod._metrics_monitor_initialized = False + + mock_spanner = self.spanner_service.mock_spanner + original_get_results = mock_spanner.get_execute_streaming_sql_results + current_time = datetime(2026, 1, 1, 0, 0, 0) + time_lock = threading.Lock() + + class _FakeDatetime(datetime): + @classmethod + def now(cls, tz=None): + with time_lock: + return current_time + + def delayed_get_results(query_sql, started_transaction): + nonlocal current_time + for message in original_get_results(query_sql, started_transaction): + with time_lock: + current_time += timedelta(milliseconds=chunk_delay_milliseconds) + yield message + + reader = InMemoryMetricReader() + meter_provider = MeterProvider(metric_readers=[reader]) + with ( + self.patched_metrics_environment(meter_provider), + mock.patch( + "google.cloud.spanner_v1.metrics.metrics_tracer.datetime", + _FakeDatetime, + ), + mock.patch.object( + mock_spanner, + "get_execute_streaming_sql_results", + side_effect=delayed_get_results, + ), + ): + yield reader + + def _assert_metrics_not_recorded(self, reader, method, check_count=True): + metrics = self.extract_metrics(reader, method=method) + self.assertNotIn("operation_latencies", metrics) + if check_count: + self.assertNotIn("operation_count", metrics) + + def _assert_operation_metrics_recorded( + self, + reader, + method, + expected_count=1, + min_latency=None, + check_operation_count=True, + expected_attempt_count=None, + ): + metrics = self.extract_metrics(reader, method=method) + self.assertIn( + "operation_latencies", + metrics, + f"Metrics: {list(metrics.keys())}", + ) + latency_points = metrics["operation_latencies"] + self.assertEqual(len(latency_points), 1) + latency_point = latency_points[0] + self.assertEqual(latency_point.count, expected_count) + if min_latency is not None: + self.assertGreaterEqual(latency_point.sum, min_latency) + + if check_operation_count: + self.assertIn("operation_count", metrics) + operation_count_points = metrics["operation_count"] + self.assertEqual(len(operation_count_points), 1) + self.assertEqual(operation_count_points[0].value, expected_count) + + if expected_attempt_count is not None: + self.assertIn("attempt_count", metrics) + self.assertEqual(metrics["attempt_count"][0].value, expected_attempt_count) + + return latency_point.sum + + def _assert_latency_unchanged(self, reader, method, expected_sum, expected_count=1): + metrics_after = self.extract_metrics(reader, method=method) + point_after = metrics_after["operation_latencies"][0] + self.assertEqual(point_after.count, expected_count) + self.assertEqual(point_after.sum, expected_sum) + + +class TestOperationMetricsIntegration( + MockServerTestBase, _OperationMetricsAssertionsMixin +): def setUp(self): super().setUp() - os.environ["SPANNER_DISABLE_BUILTIN_METRICS"] = "false" SpannerMetricsTracerFactory._metrics_tracer_factory = None client_mod._metrics_monitor_initialized = False def tearDown(self): super().tearDown() - os.environ["SPANNER_DISABLE_BUILTIN_METRICS"] = "true" SpannerMetricsTracerFactory._metrics_tracer_factory = None client_mod._metrics_monitor_initialized = False - def _setup_client_and_database(self): + def _setup_database(self): + from google.cloud.spanner_v1.testing.database_test import TestDatabase + client = Client( project="p", credentials=AnonymousCredentials(), @@ -65,34 +227,58 @@ def _setup_client_and_database(self): ), ) instance = client.instance("test-instance") - database = instance.database( + database = TestDatabase( "test-database", + instance, pool=FixedSizePool(size=10), - enable_interceptors_in_tests=True, ) database._interceptors.append(MetricsInterceptor()) - database._spanner_api = None + + def _cleanup_database(): + try: + database.close() + finally: + spanner_api = getattr(database, "_spanner_api", None) + transport = getattr(spanner_api, "transport", None) + if transport is not None: + transport.close() + + self.addCleanup(_cleanup_database) + database._pool.bind(database) # Pre-initialize pool sessions so they don't produce metric noise during queries session = database._pool.get() database._pool.put(session) - return client, database + return database + + def _run_delayed_query_and_get_latency( + self, + sql: str, + chunk_delay_milliseconds: int, + min_latency: int | None = None, + ): + with self._delayed_streaming_metrics(chunk_delay_milliseconds) as reader: + database = self._setup_database() + with database.snapshot() as snapshot: + results = snapshot.execute_sql(sql) + rows = iter(results) + first_row = next(rows) + self.assertEqual(first_row, [1]) - def _extract_metrics(self, reader, method=None): - metric_data = reader.get_metrics_data() - if not metric_data: - return {} - metrics = {} - for rm in metric_data.resource_metrics: - for sm in rm.scope_metrics: - for metric in sm.metrics: - points = list(metric.data.data_points) - if method: - points = [ - p for p in points if p.attributes.get("method") == method - ] - if points: - metrics[metric.name] = points - return metrics + recorded_sum = self._assert_operation_metrics_recorded( + reader, + method="Spanner.ExecuteStreamingSql", + min_latency=min_latency, + check_operation_count=False, + ) + + remaining_rows = list(rows) + self.assertEqual(remaining_rows, [[2], [3]]) + self._assert_latency_unchanged( + reader, + method="Spanner.ExecuteStreamingSql", + expected_sum=recorded_sum, + ) + return recorded_sum def test_operation_latency_not_recorded_before_stream_consumed(self): fields = [("col1", TypeCode.INT64)] @@ -106,30 +292,15 @@ def test_operation_latency_not_recorded_before_stream_consumed(self): reader = InMemoryMetricReader() meter_provider = MeterProvider(metric_readers=[reader]) - with ( - mock.patch( - "google.cloud.spanner_v1.metrics.metrics_tracer_factory.get_meter_provider", - return_value=meter_provider, - ), - mock.patch( - "google.cloud.spanner_v1.client.MeterProvider", - return_value=meter_provider, - ), - mock.patch( - "google.cloud.spanner_v1.client._get_spanner_emulator_host", - return_value=None, - ), - ): - _, database = self._setup_client_and_database() + with self.patched_metrics_environment(meter_provider): + database = self._setup_database() with database.snapshot() as snapshot: results = snapshot.execute_sql("select 1") # Verify that operational metrics for ExecuteStreamingSql have NOT been recorded yet - metrics_before = self._extract_metrics( + self._assert_metrics_not_recorded( reader, method="Spanner.ExecuteStreamingSql" ) - self.assertNotIn("operation_latencies", metrics_before) - self.assertNotIn("operation_count", metrics_before) # Consume the results while keeping `results` alive in local scope rows = list(results) @@ -137,22 +308,9 @@ def test_operation_latency_not_recorded_before_stream_consumed(self): # Verify that operational metrics are recorded immediately upon stream completion # even while `results` is still alive in scope (regression test for last=True). - metrics_after = self._extract_metrics( + self._assert_operation_metrics_recorded( reader, method="Spanner.ExecuteStreamingSql" ) - self.assertIn( - "operation_latencies", - metrics_after, - f"Metrics: {list(metrics_after.keys())}", - ) - points = metrics_after["operation_latencies"] - self.assertEqual(len(points), 1) - self.assertEqual(points[0].count, 1) - - self.assertIn("operation_count", metrics_after) - cnt_points = metrics_after["operation_count"] - self.assertEqual(len(cnt_points), 1) - self.assertEqual(cnt_points[0].value, 1) def test_operation_latency_measures_full_stream_duration(self): fields = [("col1", TypeCode.INT64)] @@ -163,66 +321,9 @@ def test_operation_latency_measures_full_stream_duration(self): ] partials = _make_partial_result_sets(fields, results_cfg) add_execute_streaming_sql_results("select 1", partials) - - orig_get = MockServerTestBase.spanner_service.mock_spanner.get_execute_streaming_sql_results - - def delayed_get_results(sql, started_transaction): - for msg in orig_get(sql, started_transaction): - time.sleep( - 0.03 - ) # 30ms server-side streaming delay per chunk (~90ms total) - yield msg - - reader = InMemoryMetricReader() - meter_provider = MeterProvider(metric_readers=[reader]) - - with ( - mock.patch.object( - MockServerTestBase.spanner_service.mock_spanner, - "get_execute_streaming_sql_results", - side_effect=delayed_get_results, - ), - mock.patch( - "google.cloud.spanner_v1.metrics.metrics_tracer_factory.get_meter_provider", - return_value=meter_provider, - ), - mock.patch( - "google.cloud.spanner_v1.client.MeterProvider", - return_value=meter_provider, - ), - mock.patch( - "google.cloud.spanner_v1.client._get_spanner_emulator_host", - return_value=None, - ), - ): - _, database = self._setup_client_and_database() - with database.snapshot() as snapshot: - results = snapshot.execute_sql("select 1") - rows = iter(results) - first_row = next(rows) - self.assertEqual(first_row, [1]) - - # Since _make_partial_result_sets doesn't set resume tokens, the - # entire stream (~90ms across 3 chunks) is buffered before the - # first row is yielded, and operation_latencies is already recorded. - metrics = self._extract_metrics( - reader, method="Spanner.ExecuteStreamingSql" - ) - self.assertIn("operation_latencies", metrics) - point = metrics["operation_latencies"][0] - self.assertEqual(point.count, 1) - self.assertGreaterEqual(point.sum, 75) - recorded_sum = point.sum - - # Consuming the remaining rows does not record a second time - remaining_rows = list(rows) - self.assertEqual(remaining_rows, [[2], [3]]) - metrics_after = self._extract_metrics( - reader, method="Spanner.ExecuteStreamingSql" - ) - point_after = metrics_after["operation_latencies"][0] - self.assertEqual(point_after.count, 1) - self.assertEqual(point_after.sum, recorded_sum) + self._run_delayed_query_and_get_latency( + "select 1", chunk_delay_milliseconds=30, min_latency=90 + ) def test_operation_latency_last_true_and_last_false_consistency(self): """Verifies last=True and last=False both record operation_latencies @@ -247,70 +348,15 @@ def test_operation_latency_last_true_and_last_false_consistency(self): add_execute_streaming_sql_results("select no_last", partials_no_last) add_execute_streaming_sql_results("select with_last", partials_with_last) - orig_get = MockServerTestBase.spanner_service.mock_spanner.get_execute_streaming_sql_results - - def delayed_get_results(sql, started_transaction): - for msg in orig_get(sql, started_transaction): - time.sleep(0.025) # 25ms server streaming delay per chunk (~75ms total) - yield msg - - def run_query_and_get_latency(sql): - SpannerMetricsTracerFactory._metrics_tracer_factory = None - client_mod._metrics_monitor_initialized = False - reader = InMemoryMetricReader() - meter_provider = MeterProvider(metric_readers=[reader]) - with ( - mock.patch.object( - MockServerTestBase.spanner_service.mock_spanner, - "get_execute_streaming_sql_results", - side_effect=delayed_get_results, - ), - mock.patch( - "google.cloud.spanner_v1.metrics.metrics_tracer_factory.get_meter_provider", - return_value=meter_provider, - ), - mock.patch( - "google.cloud.spanner_v1.client.MeterProvider", - return_value=meter_provider, - ), - mock.patch( - "google.cloud.spanner_v1.client._get_spanner_emulator_host", - return_value=None, - ), - ): - _, database = self._setup_client_and_database() - with database.snapshot() as snapshot: - results = snapshot.execute_sql(sql) - rows = iter(results) - first_row = next(rows) - self.assertEqual(first_row, [1]) - - # Operation completion is recorded as soon as the stream finishes - # buffering, before the remaining rows are consumed. - metrics = self._extract_metrics( - reader, method="Spanner.ExecuteStreamingSql" - ) - self.assertIn("operation_latencies", metrics) - point = metrics["operation_latencies"][0] - self.assertEqual(point.count, 1) - recorded_sum = point.sum - - remaining_rows = list(rows) - self.assertEqual(remaining_rows, [[2], [3]]) - metrics_after = self._extract_metrics( - reader, method="Spanner.ExecuteStreamingSql" - ) - point_after = metrics_after["operation_latencies"][0] - self.assertEqual(point_after.count, 1) - self.assertEqual(point_after.sum, recorded_sum) - return recorded_sum - - lat_no_last = run_query_and_get_latency("select no_last") - lat_with_last = run_query_and_get_latency("select with_last") - - # Both must reflect the ~75ms server streaming duration - self.assertGreaterEqual(lat_no_last, 60) - self.assertGreaterEqual(lat_with_last, 60) + latency_no_last = self._run_delayed_query_and_get_latency( + "select no_last", chunk_delay_milliseconds=25 + ) + latency_with_last = self._run_delayed_query_and_get_latency( + "select with_last", chunk_delay_milliseconds=25 + ) + + self.assertGreaterEqual(latency_no_last, 75) + self.assertGreaterEqual(latency_with_last, 75) def test_operation_latency_with_retries(self): add_select1_result() @@ -318,93 +364,41 @@ def test_operation_latency_with_retries(self): reader = InMemoryMetricReader() meter_provider = MeterProvider(metric_readers=[reader]) - with ( - mock.patch( - "google.cloud.spanner_v1.metrics.metrics_tracer_factory.get_meter_provider", - return_value=meter_provider, - ), - mock.patch( - "google.cloud.spanner_v1.client.MeterProvider", - return_value=meter_provider, - ), - mock.patch( - "google.cloud.spanner_v1.client._get_spanner_emulator_host", - return_value=None, - ), - ): - _, database = self._setup_client_and_database() + with self.patched_metrics_environment(meter_provider): + database = self._setup_database() with database.snapshot() as snapshot: results = snapshot.execute_sql("select 1") rows = list(results) self.assertEqual(len(rows), 1) - metrics = self._extract_metrics( - reader, method="Spanner.ExecuteStreamingSql" - ) # Should record 1 logical operation but 2 attempts - self.assertIn("operation_count", metrics) - op_cnt_point = metrics["operation_count"][0] - self.assertEqual(op_cnt_point.value, 1) - - self.assertIn("attempt_count", metrics) - att_cnt_point = metrics["attempt_count"][0] - self.assertEqual(att_cnt_point.value, 2) - - self.assertIn("operation_latencies", metrics) - op_lat_point = metrics["operation_latencies"][0] - self.assertEqual(op_lat_point.count, 1) + self._assert_operation_metrics_recorded( + reader, + method="Spanner.ExecuteStreamingSql", + expected_attempt_count=2, + ) def test_session_ping_emits_operation_metrics(self): add_select1_result() reader = InMemoryMetricReader() meter_provider = MeterProvider(metric_readers=[reader]) - with ( - mock.patch( - "google.cloud.spanner_v1.metrics.metrics_tracer_factory.get_meter_provider", - return_value=meter_provider, - ), - mock.patch( - "google.cloud.spanner_v1.client.MeterProvider", - return_value=meter_provider, - ), - mock.patch( - "google.cloud.spanner_v1.client._get_spanner_emulator_host", - return_value=None, - ), - ): - _, database = self._setup_client_and_database() + with self.patched_metrics_environment(meter_provider): + database = self._setup_database() session = database._pool.get() try: session.ping() finally: database._pool.put(session) - metrics = self._extract_metrics(reader, method="Spanner.ExecuteSql") - self.assertIn("operation_latencies", metrics) - self.assertEqual(metrics["operation_latencies"][0].count, 1) - self.assertIn("operation_count", metrics) - self.assertEqual(metrics["operation_count"][0].value, 1) + self._assert_operation_metrics_recorded(reader, method="Spanner.ExecuteSql") def test_operation_latency_streaming_read(self): reader = InMemoryMetricReader() meter_provider = MeterProvider(metric_readers=[reader]) - with ( - mock.patch( - "google.cloud.spanner_v1.metrics.metrics_tracer_factory.get_meter_provider", - return_value=meter_provider, - ), - mock.patch( - "google.cloud.spanner_v1.client.MeterProvider", - return_value=meter_provider, - ), - mock.patch( - "google.cloud.spanner_v1.client._get_spanner_emulator_host", - return_value=None, - ), - ): - _, database = self._setup_client_and_database() + with self.patched_metrics_environment(meter_provider): + database = self._setup_database() with database.snapshot() as snapshot: results = snapshot.read( table="citizens", @@ -412,15 +406,284 @@ def test_operation_latency_streaming_read(self): keyset=KeySet(all_=True), ) # Before consumption - metrics_before = self._extract_metrics( - reader, method="Spanner.StreamingRead" + self._assert_metrics_not_recorded( + reader, + method="Spanner.StreamingRead", + check_count=False, ) - self.assertNotIn("operation_latencies", metrics_before) # Consume list(results) - metrics = self._extract_metrics(reader, method="Spanner.StreamingRead") - self.assertIn("operation_latencies", metrics) - op_lat_point = metrics["operation_latencies"][0] - self.assertEqual(op_lat_point.count, 1) + self._assert_operation_metrics_recorded( + reader, + method="Spanner.StreamingRead", + check_operation_count=False, + ) + + +class TestOperationMetricsAsyncIntegration( + AsyncMockServerTestBase, _OperationMetricsAssertionsMixin +): + def setUp(self): + super().setUp() + SpannerMetricsTracerFactory._metrics_tracer_factory = None + client_mod._metrics_monitor_initialized = False + async_client_mod._metrics_monitor_initialized = False + + def tearDown(self): + super().tearDown() + SpannerMetricsTracerFactory._metrics_tracer_factory = None + client_mod._metrics_monitor_initialized = False + async_client_mod._metrics_monitor_initialized = False + + async def _setup_database(self): + client = self.create_client(disable_builtin_metrics=False) + database = await self.create_database( + client=client, + pool=AsyncFixedSizePool(size=10), + enable_interceptors_in_tests=True, + interceptors=[AsyncMetricsInterceptor()], + ) + session = await database._pool.get() + await database._pool.put(session) + return database + + async def test_async_operation_latency_not_recorded_before_stream_consumed(self): + fields = [("col1", TypeCode.INT64)] + partials = _make_partial_result_sets( + fields, + [ + {"values": ["1"], "last": True}, + ], + ) + add_execute_streaming_sql_results("select 1", partials) + reader = InMemoryMetricReader() + meter_provider = MeterProvider(metric_readers=[reader]) + + with self.patched_metrics_environment(meter_provider): + database = await self._setup_database() + async with database.snapshot() as snapshot: + results = await snapshot.execute_sql("select 1") + + self._assert_metrics_not_recorded( + reader, method="Spanner.ExecuteStreamingSql" + ) + + rows = [row async for row in results] + self.assertEqual(len(rows), 1) + + self._assert_operation_metrics_recorded( + reader, method="Spanner.ExecuteStreamingSql" + ) + + async def _run_delayed_query_and_get_latency( + self, + sql: str, + chunk_delay_milliseconds: int, + min_latency: int | None = None, + ): + with self._delayed_streaming_metrics(chunk_delay_milliseconds) as reader: + database = await self._setup_database() + return await self._consume_and_assert_streaming_latency( + database, + reader, + sql, + min_latency=min_latency, + ) + + async def _consume_and_assert_streaming_latency( + self, + database, + reader, + sql: str, + min_latency: int | None = None, + ): + async with database.snapshot() as snapshot: + results = await snapshot.execute_sql(sql) + rows = aiter(results) + first_row = await anext(rows) + self.assertEqual(first_row, [1]) + + recorded_sum = self._assert_operation_metrics_recorded( + reader, + method="Spanner.ExecuteStreamingSql", + min_latency=min_latency, + check_operation_count=False, + ) + + remaining_rows = [row async for row in rows] + self.assertEqual(remaining_rows, [[2], [3]]) + self._assert_latency_unchanged( + reader, + method="Spanner.ExecuteStreamingSql", + expected_sum=recorded_sum, + ) + return recorded_sum + + async def test_async_operation_latency_measures_full_stream_duration(self): + fields = [("col1", TypeCode.INT64)] + partials = _make_partial_result_sets( + fields, + [ + {"values": ["1"], "last": False}, + {"values": ["2"], "last": False}, + {"values": ["3"], "last": False}, + ], + ) + add_execute_streaming_sql_results("select 1", partials) + await self._run_delayed_query_and_get_latency( + "select 1", chunk_delay_milliseconds=30, min_latency=90 + ) + + async def test_async_operation_latency_last_true_and_last_false_consistency(self): + fields = [("col1", TypeCode.INT64)] + partials_no_last = _make_partial_result_sets( + fields, + [ + {"values": ["1"], "last": False}, + {"values": ["2"], "last": False}, + {"values": ["3"], "last": False}, + ], + ) + partials_with_last = _make_partial_result_sets( + fields, + [ + {"values": ["1"], "last": False}, + {"values": ["2"], "last": False}, + {"values": ["3"], "last": True}, + ], + ) + add_execute_streaming_sql_results("select no_last", partials_no_last) + add_execute_streaming_sql_results("select with_last", partials_with_last) + + latency_no_last = await self._run_delayed_query_and_get_latency( + "select no_last", chunk_delay_milliseconds=25 + ) + latency_with_last = await self._run_delayed_query_and_get_latency( + "select with_last", chunk_delay_milliseconds=25 + ) + + self.assertGreaterEqual(latency_no_last, 75) + self.assertGreaterEqual(latency_with_last, 75) + + async def test_async_operation_latency_with_retries(self): + add_select1_result() + add_error(SpannerServicer.ExecuteStreamingSql.__name__, unavailable_status()) + reader = InMemoryMetricReader() + meter_provider = MeterProvider(metric_readers=[reader]) + + with self.patched_metrics_environment(meter_provider): + database = await self._setup_database() + async with database.snapshot() as snapshot: + results = await snapshot.execute_sql("select 1") + rows = [row async for row in results] + self.assertEqual(len(rows), 1) + + self._assert_operation_metrics_recorded( + reader, + method="Spanner.ExecuteStreamingSql", + expected_attempt_count=2, + ) + + async def test_async_session_ping_emits_operation_metrics(self): + add_select1_result() + reader = InMemoryMetricReader() + meter_provider = MeterProvider(metric_readers=[reader]) + + with self.patched_metrics_environment(meter_provider): + database = await self._setup_database() + session = await database._pool.get() + try: + await session.ping() + finally: + await database._pool.put(session) + + self._assert_operation_metrics_recorded(reader, method="Spanner.ExecuteSql") + + async def test_async_operation_latency_streaming_read(self): + reader = InMemoryMetricReader() + meter_provider = MeterProvider(metric_readers=[reader]) + + with self.patched_metrics_environment(meter_provider): + database = await self._setup_database() + async with database.snapshot() as snapshot: + results = await snapshot.read( + table="citizens", + columns=["email", "first_name"], + keyset=KeySet(all_=True), + ) + self._assert_metrics_not_recorded( + reader, + method="Spanner.StreamingRead", + check_count=False, + ) + + _ = [row async for row in results] + + self._assert_operation_metrics_recorded( + reader, + method="Spanner.StreamingRead", + check_operation_count=False, + ) + + async def test_async_operation_latency_recorded_on_early_stream_close(self): + sql = "select id from early_close_table" + add_single_result( + sql, + "id", + TypeCode.INT64, + [("1",), ("2",), ("3",)], + chunk_config=PartialResultSetConfig(set_last_flag=True), + ) + reader = InMemoryMetricReader() + meter_provider = MeterProvider(metric_readers=[reader]) + + with self.patched_metrics_environment(meter_provider): + database = await self._setup_database() + async with database.snapshot() as snapshot: + results = await snapshot.execute_sql(sql) + row_iterator = aiter(results) + first_row = await anext(row_iterator) + self.assertEqual([1], first_row) + + self._assert_metrics_not_recorded( + reader, + method="Spanner.ExecuteStreamingSql", + check_count=False, + ) + + # AsyncStreamedResultSet.__aiter__ does not close _response_iterator on aclose(). + await results._response_iterator.aclose() + await row_iterator.aclose() + + self._assert_operation_metrics_recorded( + reader, method="Spanner.ExecuteStreamingSql" + ) + + async def test_async_operation_latency_recorded_on_mid_stream_error(self): + sql = "select id, payload from error_metrics_docs" + add_multi_column_result( + sql, + [("id", TypeCode.INT64), ("payload", TypeCode.STRING)], + [("1", "hello world"), ("2", "abcdef")], + chunk_config=PartialResultSetConfig(max_string_chunk_length=4), + ) + add_execute_streaming_sql_error(sql, 3, deadline_exceeded_status()) + reader = InMemoryMetricReader() + meter_provider = MeterProvider(metric_readers=[reader]) + + with self.patched_metrics_environment(meter_provider): + database = await self._setup_database() + async with database.snapshot() as snapshot: + results = await snapshot.execute_sql(sql) + row_iterator = aiter(results) + self.assertEqual([1, "hello world"], await anext(row_iterator)) + with self.assertRaisesRegex( + exceptions.DeadlineExceeded, "Deadline exceeded." + ): + await anext(row_iterator) + + self._assert_operation_metrics_recorded( + reader, method="Spanner.ExecuteStreamingSql" + ) diff --git a/packages/google-cloud-spanner/tests/mockserver_tests/test_pool_binding.py b/packages/google-cloud-spanner/tests/mockserver_tests/test_pool_binding.py index a4d14f14cc47..268b45f48041 100644 --- a/packages/google-cloud-spanner/tests/mockserver_tests/test_pool_binding.py +++ b/packages/google-cloud-spanner/tests/mockserver_tests/test_pool_binding.py @@ -19,21 +19,23 @@ FixedSizePool, PingingPool, ) -from tests.mockserver_tests.mock_server_test_base import MockServerTestBase +from google.cloud.spanner_v1._async.pool import BurstyPool as AsyncBurstyPool +from google.cloud.spanner_v1._async.pool import FixedSizePool as AsyncFixedSizePool +from google.cloud.spanner_v1._async.pool import PingingPool as AsyncPingingPool +from tests.mockserver_tests.mock_server_test_base import ( + AsyncMockServerTestBase, + MockServerTestBase, +) -class TestPoolBinding(MockServerTestBase): - def test_database_binding_of_fixed_size_pool(self): - pool = FixedSizePool( - size=1, - default_timeout=500, - database_role="parent", - ) - database = self.instance.database("fixed_size_db", pool=pool) - self.assertEqual(database._pool.database_role, "parent") +class _PoolBindingAssertionsMixin: + """Shared synchronous assertions for pool binding and role propagation.""" + + def _assert_pool_bound(self, database, expected_pool_role: str): + self.assertEqual(database._pool.database_role, expected_pool_role) self.assertIs(database._pool._database, database) - # Verify that BatchCreateSessions was called with the creator_role. + def _assert_batch_create_role(self, expected_role: str): batch_create_requests = [ request for request in self.spanner_service.requests @@ -41,17 +43,39 @@ def test_database_binding_of_fixed_size_pool(self): ] self.assertEqual(len(batch_create_requests), 1) self.assertEqual( - batch_create_requests[0].session_template.creator_role, "parent" + batch_create_requests[0].session_template.creator_role, expected_role ) - # Verify that a session checked out from the pool has the database_role - # and matches the server-side session created with creator_role. + def _assert_session_role(self, session, expected_role: str): + self.assertEqual(session.database_role, expected_role) + self.assertIn(session.name, self.spanner_service.sessions) + server_session = self.spanner_service.sessions[session.name] + self.assertEqual(server_session.creator_role, expected_role) + + def _assert_create_session_role(self, expected_role: str): + create_session_requests = [ + request + for request in self.spanner_service.requests + if isinstance(request, CreateSessionRequest) + ] + self.assertEqual(len(create_session_requests), 1) + self.assertEqual(create_session_requests[0].session.creator_role, expected_role) + + +class TestPoolBinding(_PoolBindingAssertionsMixin, MockServerTestBase): + def test_database_binding_of_fixed_size_pool(self): + pool = FixedSizePool( + size=1, + default_timeout=500, + database_role="parent", + ) + database = self.instance.database("fixed_size_db", pool=pool) + self._assert_pool_bound(database, "parent") + self._assert_batch_create_role("parent") + session = pool.get() try: - self.assertEqual(session.database_role, "parent") - self.assertIn(session.name, self.spanner_service.sessions) - server_session = self.spanner_service.sessions[session.name] - self.assertEqual(server_session.creator_role, "parent") + self._assert_session_role(session, "parent") finally: pool.put(session) @@ -63,28 +87,12 @@ def test_database_binding_of_pinging_pool(self): database_role="parent", ) database = self.instance.database("binding_db", pool=pool) - self.assertEqual(database._pool.database_role, "parent") - self.assertIs(database._pool._database, database) - - # Verify that BatchCreateSessions was called with the creator_role. - batch_create_requests = [ - request - for request in self.spanner_service.requests - if isinstance(request, BatchCreateSessionsRequest) - ] - self.assertEqual(len(batch_create_requests), 1) - self.assertEqual( - batch_create_requests[0].session_template.creator_role, "parent" - ) + self._assert_pool_bound(database, "parent") + self._assert_batch_create_role("parent") - # Verify that a session checked out from the pool has the database_role - # and matches the server-side session created with creator_role. session = pool.get() try: - self.assertEqual(session.database_role, "parent") - self.assertIn(session.name, self.spanner_service.sessions) - server_session = self.spanner_service.sessions[session.name] - self.assertEqual(server_session.creator_role, "parent") + self._assert_session_role(session, "parent") finally: pool.put(session) @@ -94,8 +102,7 @@ def test_database_binding_of_bursty_pool(self): database_role="parent", ) database = self.instance.database("bursty_db", pool=pool) - self.assertEqual(database._pool.database_role, "parent") - self.assertIs(database._pool._database, database) + self._assert_pool_bound(database, "parent") def test_database_binding_with_database_role_on_database(self): pool = FixedSizePool( @@ -105,25 +112,12 @@ def test_database_binding_with_database_role_on_database(self): database = self.instance.database( "db_role_on_database", pool=pool, database_role="parent" ) - self.assertEqual(database._pool.database_role, "parent") - self.assertIs(database._pool._database, database) - - batch_create_requests = [ - request - for request in self.spanner_service.requests - if isinstance(request, BatchCreateSessionsRequest) - ] - self.assertEqual(len(batch_create_requests), 1) - self.assertEqual( - batch_create_requests[0].session_template.creator_role, "parent" - ) + self._assert_pool_bound(database, "parent") + self._assert_batch_create_role("parent") session = pool.get() try: - self.assertEqual(session.database_role, "parent") - self.assertIn(session.name, self.spanner_service.sessions) - server_session = self.spanner_service.sessions[session.name] - self.assertEqual(server_session.creator_role, "parent") + self._assert_session_role(session, "parent") finally: pool.put(session) @@ -136,25 +130,12 @@ def test_database_binding_pinging_pool_inherits_database_role(self): database = self.instance.database( "pinging_inherits_db", pool=pool, database_role="parent" ) - self.assertEqual(database._pool.database_role, "parent") - self.assertIs(database._pool._database, database) - - batch_create_requests = [ - request - for request in self.spanner_service.requests - if isinstance(request, BatchCreateSessionsRequest) - ] - self.assertEqual(len(batch_create_requests), 1) - self.assertEqual( - batch_create_requests[0].session_template.creator_role, "parent" - ) + self._assert_pool_bound(database, "parent") + self._assert_batch_create_role("parent") session = pool.get() try: - self.assertEqual(session.database_role, "parent") - self.assertIn(session.name, self.spanner_service.sessions) - server_session = self.spanner_service.sessions[session.name] - self.assertEqual(server_session.creator_role, "parent") + self._assert_session_role(session, "parent") finally: pool.put(session) @@ -165,24 +146,12 @@ def test_database_binding_bursty_pool_inherits_database_role(self): database = self.instance.database( "bursty_inherits_db", pool=pool, database_role="parent" ) - self.assertEqual(database._pool.database_role, "parent") - self.assertIs(database._pool._database, database) + self._assert_pool_bound(database, "parent") session = pool.get() try: - self.assertEqual(session.database_role, "parent") - self.assertIn(session.name, self.spanner_service.sessions) - server_session = self.spanner_service.sessions[session.name] - self.assertEqual(server_session.creator_role, "parent") - create_session_requests = [ - request - for request in self.spanner_service.requests - if isinstance(request, CreateSessionRequest) - ] - self.assertEqual(len(create_session_requests), 1) - create_session_request = create_session_requests[0] - creator_role = create_session_request.session.creator_role - self.assertEqual(creator_role, "parent") + self._assert_session_role(session, "parent") + self._assert_create_session_role("parent") finally: pool.put(session) @@ -196,26 +165,124 @@ def test_database_binding_pool_role_precedence(self): "role_precedence_db", pool=pool, database_role="database-role" ) self.assertEqual(database.database_role, "database-role") - self.assertEqual(database._pool.database_role, "pool-role") - - batch_create_requests = [ - request - for request in self.spanner_service.requests - if isinstance(request, BatchCreateSessionsRequest) - ] - self.assertEqual(len(batch_create_requests), 1) - self.assertEqual( - batch_create_requests[0].session_template.creator_role, "pool-role" - ) + self._assert_pool_bound(database, "pool-role") + self._assert_batch_create_role("pool-role") session = pool.get() try: - self.assertEqual(session.database_role, "pool-role") - self.assertIn(session.name, self.spanner_service.sessions) - server_session = self.spanner_service.sessions[session.name] - self.assertEqual( - server_session.creator_role, - "pool-role", - ) + self._assert_session_role(session, "pool-role") finally: pool.put(session) + + +class TestPoolBindingAsync(_PoolBindingAssertionsMixin, AsyncMockServerTestBase): + async def test_async_database_binding_of_fixed_size_pool(self): + pool = AsyncFixedSizePool( + size=1, + default_timeout=500, + database_role="parent", + ) + database = await self.create_database("fixed_size_db", pool=pool) + self._assert_pool_bound(database, "parent") + self._assert_batch_create_role("parent") + + session = await pool.get() + try: + self._assert_session_role(session, "parent") + finally: + await pool.put(session) + + async def test_async_database_binding_of_pinging_pool(self): + pool = AsyncPingingPool( + size=1, + default_timeout=500, + ping_interval=100, + database_role="parent", + ) + database = await self.create_database("binding_db", pool=pool) + self._assert_pool_bound(database, "parent") + self._assert_batch_create_role("parent") + + session = await pool.get() + try: + self._assert_session_role(session, "parent") + finally: + await pool.put(session) + + async def test_async_database_binding_of_bursty_pool(self): + pool = AsyncBurstyPool( + target_size=1, + database_role="parent", + ) + database = await self.create_database("bursty_db", pool=pool) + self._assert_pool_bound(database, "parent") + + async def test_async_database_binding_with_database_role_on_database(self): + pool = AsyncFixedSizePool( + size=1, + default_timeout=500, + ) + database = await self.create_database( + "db_role_on_database", pool=pool, database_role="parent" + ) + self._assert_pool_bound(database, "parent") + self._assert_batch_create_role("parent") + + session = await pool.get() + try: + self._assert_session_role(session, "parent") + finally: + await pool.put(session) + + async def test_async_database_binding_pinging_pool_inherits_database_role(self): + pool = AsyncPingingPool( + size=1, + default_timeout=500, + ping_interval=100, + ) + database = await self.create_database( + "pinging_inherits_db", pool=pool, database_role="parent" + ) + self._assert_pool_bound(database, "parent") + self._assert_batch_create_role("parent") + + session = await pool.get() + try: + self._assert_session_role(session, "parent") + finally: + await pool.put(session) + + async def test_async_database_binding_bursty_pool_inherits_database_role(self): + pool = AsyncBurstyPool( + target_size=1, + ) + database = await self.create_database( + "bursty_inherits_db", pool=pool, database_role="parent" + ) + self._assert_pool_bound(database, "parent") + + session = await pool.get() + try: + self._assert_session_role(session, "parent") + self._assert_create_session_role("parent") + finally: + await pool.put(session) + + async def test_async_database_binding_pool_role_precedence(self): + pool = AsyncFixedSizePool( + size=1, + default_timeout=500, + database_role="pool-role", + ) + database = await self.create_database( + "role_precedence_db", pool=pool, database_role="database-role" + ) + self.assertEqual(database.database_role, "database-role") + self._assert_pool_bound(database, "pool-role") + self._assert_batch_create_role("pool-role") + + session = await pool.get() + try: + self._assert_session_role(session, "pool-role") + finally: + await pool.put(session) diff --git a/packages/google-cloud-spanner/tests/mockserver_tests/test_request_id_header.py b/packages/google-cloud-spanner/tests/mockserver_tests/test_request_id_header.py index 7498863593ae..ff398ac2d749 100644 --- a/packages/google-cloud-spanner/tests/mockserver_tests/test_request_id_header.py +++ b/packages/google-cloud-spanner/tests/mockserver_tests/test_request_id_header.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import asyncio import random import threading @@ -25,6 +26,7 @@ from google.cloud.spanner_v1.request_id_header import REQ_RAND_PROCESS_ID from google.cloud.spanner_v1.testing.mock_spanner import SpannerServicer from tests.mockserver_tests.mock_server_test_base import ( + AsyncMockServerTestBase, MockServerTestBase, aborted_status, add_error, @@ -33,64 +35,206 @@ ) -class TestRequestIDHeader(MockServerTestBase): - def tearDown(self): - super().tearDown() - self.database._x_goog_request_id_interceptor.reset() +class _RequestIDAssertionsMixin: + """Shared synchronous assertions for x-goog-spanner-request-id headers.""" - def test_snapshot_execute_sql(self): - add_select1_result() - if not getattr(self.database, "_interceptors", None): - self.database._interceptors = MockServerTestBase._interceptors - with self.database.snapshot() as snapshot: - results = snapshot.execute_sql("select 1") - result_list = [] - for row in results: - result_list.append(row) - self.assertEqual(1, row[0]) - self.assertEqual(1, len(result_list)) - requests = self.spanner_service.requests + def _get_request_id_segments(self, database): + interceptor = database._x_goog_request_id_interceptor + return interceptor.stream_request_ids, interceptor.unary_request_ids + + @staticmethod + def _count_leading_session_requests(requests) -> int: + session_requests_before = 0 + for request in requests: + if isinstance(request, (BatchCreateSessionsRequest, CreateSessionRequest)): + session_requests_before += 1 + else: + break + return session_requests_before + + def _assert_single_execute_sql_request_ids(self, database, requests): self.assert_requests_sequence( requests, [ExecuteSqlRequest], TransactionType.READ_ONLY, allow_multiple_batch_create=True, ) - NTH_CLIENT = self.database._nth_client_id - CHANNEL_ID = self.database._channel_id - got_stream_segments, got_unary_segments = self.canonicalize_request_id_headers() - # Filter out CreateSessionRequest unary segments for comparison + nth_client = database._nth_client_id + channel_id = database._channel_id + got_stream_segments, got_unary_segments = self._get_request_id_segments( + database + ) filtered_unary_segments = [ - seg for seg in got_unary_segments if not seg[0].endswith("/CreateSession") + segment + for segment in got_unary_segments + if not segment[0].endswith("/CreateSession") ] want_unary_segments = [ ( "/google.spanner.v1.Spanner/BatchCreateSessions", - (1, REQ_RAND_PROCESS_ID, NTH_CLIENT, CHANNEL_ID, 1, 1), + (1, REQ_RAND_PROCESS_ID, nth_client, channel_id, 1, 1), ) ] - # Dynamically determine the expected sequence number for ExecuteStreamingSql - session_requests_before = 0 - for req in requests: - if isinstance(req, (BatchCreateSessionsRequest, CreateSessionRequest)): - session_requests_before += 1 - elif isinstance(req, ExecuteSqlRequest): - break + session_requests_before = self._count_leading_session_requests(requests) want_stream_segments = [ ( "/google.spanner.v1.Spanner/ExecuteStreamingSql", ( 1, REQ_RAND_PROCESS_ID, - NTH_CLIENT, - CHANNEL_ID, + nth_client, + channel_id, 1 + session_requests_before, 1, ), ) ] - assert filtered_unary_segments == want_unary_segments - assert got_stream_segments == want_stream_segments + self.assertEqual(want_unary_segments, filtered_unary_segments) + self.assertEqual(want_stream_segments, got_stream_segments) + + def _assert_concurrent_execute_sql_request_ids( + self, database, requests, concurrency + ): + # 1 warmup query + `concurrency` concurrent queries + expected_queries = concurrency + 1 + expected_min = 1 + expected_queries + expected_max = expected_min + 1 + self.assertTrue( + expected_min <= len(requests) <= expected_max, + f"Expected {expected_min} or {expected_max} requests, got {len(requests)}: {requests}", + ) + client_id = database._nth_client_id + channel_id = database._channel_id + got_stream_segments, got_unary_segments = self._get_request_id_segments( + database + ) + want_unary_segment = ( + "/google.spanner.v1.Spanner/BatchCreateSessions", + (1, REQ_RAND_PROCESS_ID, client_id, channel_id, 1, 1), + ) + self.assertTrue( + any(segment == want_unary_segment for segment in got_unary_segments) + ) + + session_requests_before = self._count_leading_session_requests(requests) + want_stream_segments = [ + ( + "/google.spanner.v1.Spanner/ExecuteStreamingSql", + ( + 1, + REQ_RAND_PROCESS_ID, + client_id, + channel_id, + session_requests_before + index, + 1, + ), + ) + for index in range(1, expected_queries + 1) + ] + self.assertEqual(sorted(want_stream_segments), sorted(got_stream_segments)) + + def _assert_partitioned_dml_request_ids(self, database, requests): + self.assert_requests_sequence( + requests, + [BeginTransactionRequest, ExecuteSqlRequest], + TransactionType.PARTITIONED, + allow_multiple_batch_create=True, + ) + got_stream_segments, got_unary_segments = self._get_request_id_segments( + database + ) + nth_client = database._nth_client_id + channel_id = database._channel_id + filtered_unary_segments = [ + segment + for segment in got_unary_segments + if not segment[0].endswith("/CreateSession") + ] + session_requests_before = self._count_leading_session_requests(requests) + begin_transaction_seq = 1 + session_requests_before + execute_sql_seq = 2 + session_requests_before + want_unary_segments = [ + ( + "/google.spanner.v1.Spanner/BatchCreateSessions", + (1, REQ_RAND_PROCESS_ID, nth_client, channel_id, 1, 1), + ), + ( + "/google.spanner.v1.Spanner/BeginTransaction", + ( + 1, + REQ_RAND_PROCESS_ID, + nth_client, + channel_id, + begin_transaction_seq, + 1, + ), + ), + ] + want_stream_segments = [ + ( + "/google.spanner.v1.Spanner/ExecuteStreamingSql", + (1, REQ_RAND_PROCESS_ID, nth_client, channel_id, execute_sql_seq, 1), + ) + ] + self.assertTrue( + all(segment in filtered_unary_segments for segment in want_unary_segments) + ) + self.assertEqual(want_stream_segments, got_stream_segments) + + def _assert_unary_retry_request_ids(self, database, requests): + self.assert_requests_sequence( + requests, + [ExecuteSqlRequest], + TransactionType.READ_ONLY, + allow_multiple_batch_create=True, + ) + nth_client = database._nth_client_id + channel_id = database._channel_id + got_stream_segments, got_unary_segments = self._get_request_id_segments( + database + ) + expected_batch_create_segment = ( + "/google.spanner.v1.Spanner/BatchCreateSessions", + (1, REQ_RAND_PROCESS_ID, nth_client, channel_id, 1, 1), + ) + batch_create_segments = [ + segment + for segment in got_unary_segments + if segment[0].endswith("/BatchCreateSessions") + ] + self.assertEqual([expected_batch_create_segment] * 2, batch_create_segments) + has_create_session = any( + segment[0].endswith("/CreateSession") for segment in got_unary_segments + ) + execute_sql_seq = 3 if has_create_session else 2 + want_stream_segments = [ + ( + "/google.spanner.v1.Spanner/ExecuteStreamingSql", + (1, REQ_RAND_PROCESS_ID, nth_client, channel_id, execute_sql_seq, 1), + ) + ] + self.assertEqual(want_stream_segments, got_stream_segments) + + +class TestRequestIDHeader(_RequestIDAssertionsMixin, MockServerTestBase): + def tearDown(self): + super().tearDown() + self.database._x_goog_request_id_interceptor.reset() + + def test_snapshot_execute_sql(self): + add_select1_result() + if not getattr(self.database, "_interceptors", None): + self.database._interceptors = MockServerTestBase._interceptors + with self.database.snapshot() as snapshot: + results = snapshot.execute_sql("select 1") + result_list = [] + for row in results: + result_list.append(row) + self.assertEqual(1, row[0]) + self.assertEqual(1, len(result_list)) + self._assert_single_execute_sql_request_ids( + self.database, self.spanner_service.requests + ) def test_snapshot_read_concurrent(self): add_select1_result() @@ -118,46 +262,9 @@ def select1(): random.shuffle(threads) for thread in threads: thread.join() - requests = self.spanner_service.requests - # Allow for an extra request due to multiplexed session creation - expected_min = 2 + n - expected_max = expected_min + 1 - assert expected_min <= len(requests) <= expected_max, ( - f"Expected {expected_min} or {expected_max} requests, got {len(requests)}: {requests}" + self._assert_concurrent_execute_sql_request_ids( + db, self.spanner_service.requests, n ) - client_id = db._nth_client_id - channel_id = db._channel_id - got_stream_segments, got_unary_segments = self.canonicalize_request_id_headers() - want_unary_segments = [ - ( - "/google.spanner.v1.Spanner/BatchCreateSessions", - (1, REQ_RAND_PROCESS_ID, client_id, channel_id, 1, 1), - ), - ] - assert any(seg == want_unary_segments[0] for seg in got_unary_segments) - - # Dynamically determine the expected sequence numbers for ExecuteStreamingSql - session_requests_before = 0 - for req in requests: - if isinstance(req, (BatchCreateSessionsRequest, CreateSessionRequest)): - session_requests_before += 1 - elif isinstance(req, ExecuteSqlRequest): - break - want_stream_segments = [ - ( - "/google.spanner.v1.Spanner/ExecuteStreamingSql", - ( - 1, - REQ_RAND_PROCESS_ID, - client_id, - channel_id, - session_requests_before + i, - 1, - ), - ) - for i in range(1, n + 2) - ] - assert sorted(got_stream_segments) == sorted(want_stream_segments) def test_database_run_in_transaction_retries_on_abort(self): counters = dict(aborted=0) @@ -183,58 +290,32 @@ def test_database_execute_partitioned_dml_request_id(self): if not getattr(self.database, "_interceptors", None): self.database._interceptors = MockServerTestBase._interceptors _ = self.database.execute_partitioned_dml("select 1") - requests = self.spanner_service.requests - self.assert_requests_sequence( - requests, - [BeginTransactionRequest, ExecuteSqlRequest], - TransactionType.PARTITIONED, - allow_multiple_batch_create=True, + self._assert_partitioned_dml_request_ids( + self.database, self.spanner_service.requests ) - got_stream_segments, got_unary_segments = self.canonicalize_request_id_headers() - NTH_CLIENT = self.database._nth_client_id - CHANNEL_ID = self.database._channel_id - # Allow for extra unary segments due to session creation - filtered_unary_segments = [ - seg for seg in got_unary_segments if not seg[0].endswith("/CreateSession") - ] - # Find the actual sequence number for BeginTransaction - begin_txn_seq = None - for seg in filtered_unary_segments: - if seg[0].endswith("/BeginTransaction"): - begin_txn_seq = seg[1][4] - break - want_unary_segments = [ - ( - "/google.spanner.v1.Spanner/BatchCreateSessions", - (1, REQ_RAND_PROCESS_ID, NTH_CLIENT, CHANNEL_ID, 1, 1), - ), - ( - "/google.spanner.v1.Spanner/BeginTransaction", - (1, REQ_RAND_PROCESS_ID, NTH_CLIENT, CHANNEL_ID, begin_txn_seq, 1), - ), - ] - # Dynamically determine the expected sequence number for ExecuteStreamingSql - session_requests_before = 0 - for req in requests: - if isinstance(req, (BatchCreateSessionsRequest, CreateSessionRequest)): - session_requests_before += 1 - elif isinstance(req, ExecuteSqlRequest): - break - # Find the actual sequence number for ExecuteStreamingSql - exec_sql_seq = got_stream_segments[0][1][4] if got_stream_segments else None - want_stream_segments = [ - ( - "/google.spanner.v1.Spanner/ExecuteStreamingSql", - (1, REQ_RAND_PROCESS_ID, NTH_CLIENT, CHANNEL_ID, exec_sql_seq, 1), - ) - ] - assert all(seg in filtered_unary_segments for seg in want_unary_segments) - assert got_stream_segments == want_stream_segments def test_unary_retryable_error(self): add_select1_result() add_error(SpannerServicer.BatchCreateSessions.__name__, unavailable_status()) + if not getattr(self.database, "_interceptors", None): + self.database._interceptors = MockServerTestBase._interceptors + with self.database.snapshot() as snapshot: + results = snapshot.execute_sql("select 1") + result_list = [] + for row in results: + result_list.append(row) + self.assertEqual(1, row[0]) + self.assertEqual(1, len(result_list)) + + self._assert_unary_retry_request_ids( + self.database, self.spanner_service.requests + ) + + def test_streaming_retryable_error(self): + add_select1_result() + add_error(SpannerServicer.ExecuteStreamingSql.__name__, unavailable_status()) + if not getattr(self.database, "_interceptors", None): self.database._interceptors = MockServerTestBase._interceptors with self.database.snapshot() as snapshot: @@ -248,36 +329,148 @@ def test_unary_retryable_error(self): requests = self.spanner_service.requests self.assert_requests_sequence( requests, - [ExecuteSqlRequest], + [ExecuteSqlRequest, ExecuteSqlRequest], TransactionType.READ_ONLY, allow_multiple_batch_create=True, ) - NTH_CLIENT = self.database._nth_client_id - CHANNEL_ID = self.database._channel_id - # Now ensure monotonicity of the received request-id segments. - got_stream_segments, got_unary_segments = self.canonicalize_request_id_headers() + def canonicalize_request_id_headers(self): + return self._get_request_id_segments(self.database) - # Dynamically determine the expected sequence number for ExecuteStreamingSql - exec_sql_seq = got_stream_segments[0][1][4] if got_stream_segments else None - want_stream_segments = [ - ( - "/google.spanner.v1.Spanner/ExecuteStreamingSql", - (1, REQ_RAND_PROCESS_ID, NTH_CLIENT, CHANNEL_ID, exec_sql_seq, 1), + +class TestRequestIDHeaderAsync(_RequestIDAssertionsMixin, AsyncMockServerTestBase): + async def get_database(self): + if self._database is None: + self._database = await self.create_database( + enable_interceptors_in_tests=True ) + return self._database + + async def test_async_snapshot_execute_sql(self): + add_select1_result() + database = await self.get_database() + async with database.snapshot() as snapshot: + results = await snapshot.execute_sql("select 1") + result_list = [] + async for row in results: + result_list.append(row) + self.assertEqual(1, row[0]) + self.assertEqual(1, len(result_list)) + self._assert_single_execute_sql_request_ids( + database, self.spanner_service.requests + ) + + async def test_async_snapshot_read_concurrent(self): + add_select1_result() + database = await self.get_database() + async with database.snapshot() as snapshot: + rows = await snapshot.execute_sql("select 1") + async for _ in rows: + pass + + async def select1(): + async with database.snapshot() as snapshot: + rows = await snapshot.execute_sql("select 1") + result_list = [] + async for row in rows: + self.assertEqual(1, row[0]) + result_list.append(row) + self.assertEqual(1, len(result_list)) + + concurrency = 10 + await asyncio.wait_for( + asyncio.gather(*(select1() for _ in range(concurrency))), + timeout=5.0, + ) + self._assert_concurrent_execute_sql_request_ids( + database, self.spanner_service.requests, concurrency + ) + + async def test_async_database_run_in_transaction_retries_on_abort(self): + counters = {"aborted": 0} + want_failed_attempts = 2 + + async def select_in_transaction(transaction): + results = await transaction.execute_sql("select 1") + async for _ in results: + pass + + if counters["aborted"] < want_failed_attempts: + counters["aborted"] += 1 + add_error(SpannerServicer.Commit.__name__, aborted_status()) + + add_select1_result() + database = await self.get_database() + await database.run_in_transaction(select_in_transaction) + got_stream_segments, got_unary_segments = self._get_request_id_segments( + database + ) + stream_segments = [ + segment + for segment in got_stream_segments + if segment[0].endswith("/ExecuteStreamingSql") ] - assert got_stream_segments == want_stream_segments + commit_segments = [ + segment for segment in got_unary_segments if segment[0].endswith("/Commit") + ] + expected_attempts = want_failed_attempts + 1 + self.assertEqual(expected_attempts, len(stream_segments)) + self.assertEqual(expected_attempts, len(commit_segments)) - def test_streaming_retryable_error(self): + expected_prefix = ( + 1, + REQ_RAND_PROCESS_ID, + database._nth_client_id, + database._channel_id, + ) + session_requests_before = self._count_leading_session_requests( + self.spanner_service.requests + ) + for attempt_index, (stream_segment, commit_segment) in enumerate( + zip(stream_segments, commit_segments) + ): + execute_seq = session_requests_before + 1 + 2 * attempt_index + self.assertEqual( + (*expected_prefix, execute_seq, 1), + stream_segment[1], + ) + self.assertEqual( + (*expected_prefix, execute_seq + 1, 1), + commit_segment[1], + ) + + async def test_async_database_execute_partitioned_dml_request_id(self): + add_select1_result() + database = await self.get_database() + _ = await database.execute_partitioned_dml("select 1") + self._assert_partitioned_dml_request_ids( + database, self.spanner_service.requests + ) + + async def test_async_unary_retryable_error(self): + add_select1_result() + add_error(SpannerServicer.BatchCreateSessions.__name__, unavailable_status()) + + database = await self.get_database() + async with database.snapshot() as snapshot: + results = await snapshot.execute_sql("select 1") + result_list = [] + async for row in results: + result_list.append(row) + self.assertEqual(1, row[0]) + self.assertEqual(1, len(result_list)) + + self._assert_unary_retry_request_ids(database, self.spanner_service.requests) + + async def test_async_streaming_retryable_error(self): add_select1_result() add_error(SpannerServicer.ExecuteStreamingSql.__name__, unavailable_status()) - if not getattr(self.database, "_interceptors", None): - self.database._interceptors = MockServerTestBase._interceptors - with self.database.snapshot() as snapshot: - results = snapshot.execute_sql("select 1") + database = await self.get_database() + async with database.snapshot() as snapshot: + results = await snapshot.execute_sql("select 1") result_list = [] - for row in results: + async for row in results: result_list.append(row) self.assertEqual(1, row[0]) self.assertEqual(1, len(result_list)) @@ -289,7 +482,3 @@ def test_streaming_retryable_error(self): TransactionType.READ_ONLY, allow_multiple_batch_create=True, ) - - def canonicalize_request_id_headers(self): - src = self.database._x_goog_request_id_interceptor - return src._stream_req_segments, src._unary_req_segments