From 851c4e7f66f892b173ef144f95d60350b2aa0d88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Knut=20Olav=20L=C3=B8ite?= Date: Sat, 21 Nov 2026 10:22:43 +0100 Subject: [PATCH] chore(spanner): bound streaming result buffer and disable unsafe retries Previously, streaming reads and queries buffered incoming chunks without a limit while waiting for a resume token from Spanner. If a stream sent many chunks without a resume token, memory usage could grow unboundedly and row delivery to the caller would stall. With this change: - Streaming results now buffer at most 512 chunks while waiting for a resume token. - Once that limit is exceeded, buffered chunks are flushed immediately to the caller and retries are disabled so already-yielded rows cannot be duplicated. - Subsequent chunks without a resume token continue streaming directly without buffering, and any transient stream failure is propagated immediately instead of being retried. - Retries are re-enabled only if a newer resume token arrives later in the stream. - Both the synchronous and asynchronous clients are updated consistently, with unit and mock-server tests covering the threshold, failure, and recovery paths. --- .../cloud/spanner_v1/_async/snapshot.py | 46 +- .../google/cloud/spanner_v1/snapshot.py | 38 +- .../tests/mockserver_tests/test_basics.py | 190 +++++++ .../tests/unit/_async/test_snapshot.py | 534 ++++++++++++++++++ .../tests/unit/test_snapshot.py | 435 ++++++++++++++ 5 files changed, 1213 insertions(+), 30 deletions(-) diff --git a/packages/google-cloud-spanner/google/cloud/spanner_v1/_async/snapshot.py b/packages/google-cloud-spanner/google/cloud/spanner_v1/_async/snapshot.py index cc82d35de7cf..7ffed7dba90f 100644 --- a/packages/google-cloud-spanner/google/cloud/spanner_v1/_async/snapshot.py +++ b/packages/google-cloud-spanner/google/cloud/spanner_v1/_async/snapshot.py @@ -76,6 +76,12 @@ _TRANSACTION_BEGIN_TIMEOUT_SECONDS = 30.0 +# Maximum number of PartialResultSet chunks to buffer while waiting for a +# resume_token before flushing downstream and disabling retries until a newer +# resume_token arrives. +_MAX_BUFFERED_CHUNKS = 512 + + @CrossSync.convert async def _restart_on_unavailable( method, @@ -89,6 +95,7 @@ async def _restart_on_unavailable( observability_options=None, request_id_manager=None, resource_info=None, + max_buffer_size: Optional[int] = None, ): """Restart iteration after :exc:`.ServiceUnavailable`. @@ -104,10 +111,22 @@ async def _restart_on_unavailable( :type transaction_selector: :class:`transaction_pb2.TransactionSelector` :param transaction_selector: Transaction selector object to be used in request if transaction is not passed, if both transaction_selector and transaction are passed, then transaction is given priority. + + :type max_buffer_size: int + :param max_buffer_size: + (Optional) Maximum number of ``PartialResultSet`` items to buffer + while waiting for a ``resume_token``. Defaults to + ``_MAX_BUFFERED_CHUNKS``. """ + if max_buffer_size is None: + max_buffer_size = _MAX_BUFFERED_CHUNKS + elif max_buffer_size < 0: + raise ValueError("max_buffer_size must be non-negative") + resume_token: bytes = b"" item_buffer: List[PartialResultSet] = [] + safe_to_retry: bool = True if transaction is not None: transaction_selector = transaction._build_transaction_selector_pb() @@ -199,35 +218,32 @@ async def _restart_on_unavailable( iterator = None break - if item.resume_token: + if item.resume_token and item.resume_token != resume_token: resume_token = item.resume_token + safe_to_retry = True + break + + if not safe_to_retry or len(item_buffer) > max_buffer_size: + # Flush without a resume_token; retries remain unsafe + # until a newer resume_token supersedes the flushed items. + safe_to_retry = False break else: stream_finished = True - except ServiceUnavailable: - del item_buffer[:] - request.resume_token = resume_token - if transaction is not None: - transaction_selector = transaction._build_transaction_selector_pb() - request.transaction = transaction_selector - attempt += 1 - iterator = None - continue - - except InternalServerError as exc: - resumable_error = any( + except (ServiceUnavailable, InternalServerError) as exc: + is_retryable = isinstance(exc, ServiceUnavailable) or any( resumable_message in exc.message for resumable_message in _STREAM_RESUMPTION_INTERNAL_ERROR_MESSAGES ) - if not resumable_error: + if not safe_to_retry or not is_retryable: raise _augment_error_with_request_id(exc, current_request_id) del item_buffer[:] request.resume_token = resume_token if transaction is not None: transaction_selector = transaction._build_transaction_selector_pb() - attempt += 1 request.transaction = transaction_selector + attempt += 1 iterator = None continue diff --git a/packages/google-cloud-spanner/google/cloud/spanner_v1/snapshot.py b/packages/google-cloud-spanner/google/cloud/spanner_v1/snapshot.py index 3ba9b71014d1..9f9b6bbc0180 100644 --- a/packages/google-cloud-spanner/google/cloud/spanner_v1/snapshot.py +++ b/packages/google-cloud-spanner/google/cloud/spanner_v1/snapshot.py @@ -74,6 +74,7 @@ "Received unexpected EOS on DATA frame from server", ) _TRANSACTION_BEGIN_TIMEOUT_SECONDS = 30.0 +_MAX_BUFFERED_CHUNKS = 512 def _restart_on_unavailable( @@ -88,6 +89,7 @@ def _restart_on_unavailable( observability_options=None, request_id_manager=None, resource_info=None, + max_buffer_size: Optional[int] = None, ): """Restart iteration after :exc:`.ServiceUnavailable`. @@ -102,9 +104,20 @@ def _restart_on_unavailable( :type transaction_selector: :class:`transaction_pb2.TransactionSelector` :param transaction_selector: Transaction selector object to be used in request if transaction is not passed, - if both transaction_selector and transaction are passed, then transaction is given priority.""" + if both transaction_selector and transaction are passed, then transaction is given priority. + + :type max_buffer_size: int + :param max_buffer_size: + (Optional) Maximum number of ``PartialResultSet`` items to buffer + while waiting for a ``resume_token``. Defaults to + ``_MAX_BUFFERED_CHUNKS``.""" + if max_buffer_size is None: + max_buffer_size = _MAX_BUFFERED_CHUNKS + elif max_buffer_size < 0: + raise ValueError("max_buffer_size must be non-negative") resume_token: bytes = b"" item_buffer: List[PartialResultSet] = [] + safe_to_retry: bool = True if transaction is not None: transaction_selector = transaction._build_transaction_selector_pb() elif transaction_selector is None: @@ -176,35 +189,30 @@ def _restart_on_unavailable( _drain_stream(iterator) iterator = None break - if item.resume_token: + if item.resume_token and item.resume_token != resume_token: resume_token = item.resume_token + safe_to_retry = True + break + if not safe_to_retry or len(item_buffer) > max_buffer_size: + safe_to_retry = False break else: stream_finished = True - except ServiceUnavailable: - del item_buffer[:] - request.resume_token = resume_token - if transaction is not None: - transaction_selector = transaction._build_transaction_selector_pb() - request.transaction = transaction_selector - attempt += 1 - iterator = None - continue - except InternalServerError as exc: - resumable_error = any( + except (ServiceUnavailable, InternalServerError) as exc: + is_retryable = isinstance(exc, ServiceUnavailable) or any( ( resumable_message in exc.message for resumable_message in _STREAM_RESUMPTION_INTERNAL_ERROR_MESSAGES ) ) - if not resumable_error: + if not safe_to_retry or not is_retryable: raise _augment_error_with_request_id(exc, current_request_id) del item_buffer[:] request.resume_token = resume_token if transaction is not None: transaction_selector = transaction._build_transaction_selector_pb() - attempt += 1 request.transaction = transaction_selector + attempt += 1 iterator = None continue except Exception as exc: diff --git a/packages/google-cloud-spanner/tests/mockserver_tests/test_basics.py b/packages/google-cloud-spanner/tests/mockserver_tests/test_basics.py index 1eae77319b43..c12e23464fa0 100644 --- a/packages/google-cloud-spanner/tests/mockserver_tests/test_basics.py +++ b/packages/google-cloud-spanner/tests/mockserver_tests/test_basics.py @@ -676,6 +676,97 @@ def test_partial_result_set_without_resume_tokens_and_explicit_chunked_value(sel [b"", b"mid_tok"], [request.resume_token for request in execute_requests] ) + def test_partial_result_set_buffer_limit_safe_to_retry(self): + from google.cloud.spanner_v1.snapshot import _MAX_BUFFERED_CHUNKS + + # 1. Exactly 512 tokenless chunks stay within the threshold and retry cleanly. + sql_within = "select id from buffer_limit_within" + within_rows = [(str(index),) for index in range(_MAX_BUFFERED_CHUNKS)] + add_multi_column_result( + sql_within, + [("id", TypeCode.INT64)], + within_rows, + chunk_config=PartialResultSetConfig( + max_values_per_chunk=1, emit_resume_tokens=False + ), + ) + add_execute_streaming_sql_error( + sql_within, _MAX_BUFFERED_CHUNKS, unavailable_status() + ) + with self.database.snapshot() as snapshot: + rows = list(snapshot.execute_sql(sql_within)) + self.assertEqual([[index] for index in range(_MAX_BUFFERED_CHUNKS)], rows) + execute_requests = _requests_of_type(self.spanner_service, ExecuteSqlRequest) + self.assertEqual( + [b"", b""], [request.resume_token for request in execute_requests] + ) + + # 2. 513 tokenless chunks exceed the threshold, flush downstream, and fail + # without retrying on subsequent UNAVAILABLE. + sql_exceeded = "select id from buffer_limit_exceeded" + exceeded_rows = [(str(index),) for index in range(_MAX_BUFFERED_CHUNKS + 1)] + add_multi_column_result( + sql_exceeded, + [("id", TypeCode.INT64)], + exceeded_rows, + chunk_config=PartialResultSetConfig( + max_values_per_chunk=1, emit_resume_tokens=False + ), + ) + self.spanner_service.clear_requests() + add_execute_streaming_sql_error( + sql_exceeded, _MAX_BUFFERED_CHUNKS + 1, unavailable_status() + ) + collected_rows = [] + with self.database.snapshot() as snapshot: + with self.assertRaisesRegex( + exceptions.ServiceUnavailable, "Service unavailable." + ): + for row in snapshot.execute_sql(sql_exceeded): + collected_rows.append(row) + self.assertEqual( + [[index] for index in range(_MAX_BUFFERED_CHUNKS + 1)], collected_rows + ) + execute_requests = _requests_of_type(self.spanner_service, ExecuteSqlRequest) + self.assertEqual([b""], [request.resume_token for request in execute_requests]) + + # 3. Exceeding the threshold followed by a newer resume_token re-enables retry. + sql_recovered = "select id from buffer_limit_recovered" + partial_specs = [ + {"values": [str(index)]} for index in range(_MAX_BUFFERED_CHUNKS + 1) + ] + partial_specs.append( + { + "values": [str(_MAX_BUFFERED_CHUNKS + 1)], + "resume_token": b"tok_recover", + } + ) + partial_specs.append({"values": [str(_MAX_BUFFERED_CHUNKS + 2)]}) + partial_specs.append( + { + "values": [str(_MAX_BUFFERED_CHUNKS + 3)], + "last": True, + } + ) + add_execute_streaming_sql_results( + sql_recovered, + _make_partial_result_sets([("id", TypeCode.INT64)], partial_specs), + ) + self.spanner_service.clear_requests() + add_execute_streaming_sql_error( + sql_recovered, _MAX_BUFFERED_CHUNKS + 3, unavailable_status() + ) + with self.database.snapshot() as snapshot: + rows = list(snapshot.execute_sql(sql_recovered)) + self.assertEqual( + [[index] for index in range(_MAX_BUFFERED_CHUNKS + 4)], rows + ) + execute_requests = _requests_of_type(self.spanner_service, ExecuteSqlRequest) + self.assertEqual( + [b"", b"tok_recover"], + [request.resume_token for request in execute_requests], + ) + def test_partial_result_set_config_validation(self): # Verify PartialResultSetConfig rejects non-positive chunking thresholds. with self.assertRaisesRegex( @@ -1199,6 +1290,105 @@ async def test_async_execute_streaming_sql_last_flag_trailing_error_drained(self rows = [row async for row in results] self.assertEqual([[10], [20]], rows) + async def test_async_partial_result_set_buffer_limit_safe_to_retry(self): + from google.cloud.spanner_v1._async.snapshot import _MAX_BUFFERED_CHUNKS + + # 1. Exactly 512 tokenless chunks stay within the threshold and retry cleanly. + sql_within = "select id from async_buffer_limit_within" + within_rows = [(str(index),) for index in range(_MAX_BUFFERED_CHUNKS)] + add_multi_column_result( + sql_within, + [("id", TypeCode.INT64)], + within_rows, + chunk_config=PartialResultSetConfig( + max_values_per_chunk=1, emit_resume_tokens=False + ), + ) + add_execute_streaming_sql_error( + sql_within, _MAX_BUFFERED_CHUNKS, unavailable_status() + ) + async with self._async_database() as database: + async with database.snapshot() as snapshot: + results = await snapshot.execute_sql(sql_within) + rows = [row async for row in results] + self.assertEqual( + [[index] for index in range(_MAX_BUFFERED_CHUNKS)], rows + ) + execute_requests = _requests_of_type(self.spanner_service, ExecuteSqlRequest) + self.assertEqual( + [b"", b""], [request.resume_token for request in execute_requests] + ) + + # 2. 513 tokenless chunks exceed the threshold, flush downstream, and fail + # without retrying on subsequent UNAVAILABLE. + sql_exceeded = "select id from async_buffer_limit_exceeded" + exceeded_rows = [(str(index),) for index in range(_MAX_BUFFERED_CHUNKS + 1)] + add_multi_column_result( + sql_exceeded, + [("id", TypeCode.INT64)], + exceeded_rows, + chunk_config=PartialResultSetConfig( + max_values_per_chunk=1, emit_resume_tokens=False + ), + ) + self.spanner_service.clear_requests() + add_execute_streaming_sql_error( + sql_exceeded, _MAX_BUFFERED_CHUNKS + 1, unavailable_status() + ) + collected_rows = [] + async with self._async_database() as database: + async with database.snapshot() as snapshot: + results = await snapshot.execute_sql(sql_exceeded) + with self.assertRaisesRegex( + exceptions.ServiceUnavailable, "Service unavailable." + ): + async for row in results: + collected_rows.append(row) + self.assertEqual( + [[index] for index in range(_MAX_BUFFERED_CHUNKS + 1)], collected_rows + ) + execute_requests = _requests_of_type(self.spanner_service, ExecuteSqlRequest) + self.assertEqual([b""], [request.resume_token for request in execute_requests]) + + # 3. Exceeding the threshold followed by a newer resume_token re-enables retry. + sql_recovered = "select id from async_buffer_limit_recovered" + partial_specs = [ + {"values": [str(index)]} for index in range(_MAX_BUFFERED_CHUNKS + 1) + ] + partial_specs.append( + { + "values": [str(_MAX_BUFFERED_CHUNKS + 1)], + "resume_token": b"tok_recover", + } + ) + partial_specs.append({"values": [str(_MAX_BUFFERED_CHUNKS + 2)]}) + partial_specs.append( + { + "values": [str(_MAX_BUFFERED_CHUNKS + 3)], + "last": True, + } + ) + add_execute_streaming_sql_results( + sql_recovered, + _make_partial_result_sets([("id", TypeCode.INT64)], partial_specs), + ) + self.spanner_service.clear_requests() + add_execute_streaming_sql_error( + sql_recovered, _MAX_BUFFERED_CHUNKS + 3, unavailable_status() + ) + async with self._async_database() as database: + async with database.snapshot() as snapshot: + results = await snapshot.execute_sql(sql_recovered) + rows = [row async for row in results] + self.assertEqual( + [[index] for index in range(_MAX_BUFFERED_CHUNKS + 4)], rows + ) + execute_requests = _requests_of_type(self.spanner_service, ExecuteSqlRequest) + self.assertEqual( + [b"", b"tok_recover"], + [request.resume_token for request in execute_requests], + ) + def _execute_query(transaction: Transaction, sql: str): rows = transaction.execute_sql(sql, last_statement=True) diff --git a/packages/google-cloud-spanner/tests/unit/_async/test_snapshot.py b/packages/google-cloud-spanner/tests/unit/_async/test_snapshot.py index dd8ee834c37b..1dc86f25d961 100644 --- a/packages/google-cloud-spanner/tests/unit/_async/test_snapshot.py +++ b/packages/google-cloud-spanner/tests/unit/_async/test_snapshot.py @@ -618,6 +618,540 @@ async def test_restart_on_unavailable_retry_before_last(self): mock_drain.assert_called_once_with(stream_two) stream_two.cancel.assert_not_called() + async def test_restart_on_unavailable_negative_max_buffer_size(self): + from google.cloud.spanner_v1._async.snapshot import _restart_on_unavailable + + restart = mock.Mock() + request = mock.Mock() + request.transaction = None + request.resume_token = b"" + session = _Session() + snapshot = self._make_snapshot(session) + + resumable = _restart_on_unavailable( + restart, + request, + metadata=None, + trace_name="span", + session=session, + attributes={}, + transaction=snapshot, + request_id_manager=session._database, + max_buffer_size=-1, + ) + with self.assertRaisesRegex(ValueError, "max_buffer_size must be non-negative"): + async for _ in resumable: + pass + restart.assert_not_called() + + async def test_restart_on_unavailable_buffer_limit_within_threshold_retries(self): + from google.cloud.spanner_v1._async.snapshot import _restart_on_unavailable + from google.cloud.spanner_v1.types.result_set import PartialResultSet + + discarded_chunk_one = PartialResultSet(last=False) + discarded_chunk_two = PartialResultSet(last=False) + recovered_chunk = PartialResultSet(last=True) + + stream_one = _MockIterator( + discarded_chunk_one, + discarded_chunk_two, + fail_after=True, + error=ServiceUnavailable("transient"), + ) + stream_two = _MockIterator(recovered_chunk) + restart = mock.Mock(side_effect=[stream_one, stream_two]) + request = mock.Mock() + request.transaction = None + request.resume_token = b"" + session = _Session() + snapshot = self._make_snapshot(session) + + resumable = _restart_on_unavailable( + restart, + request, + metadata=None, + trace_name="span", + session=session, + attributes={}, + transaction=snapshot, + request_id_manager=session._database, + max_buffer_size=2, + ) + received_items = [received async for received in resumable] + + self.assertEqual(received_items, [recovered_chunk]) + self.assertEqual(len(restart.mock_calls), 2) + self.assertEqual(request.resume_token, b"") + + async def test_restart_on_unavailable_buffer_limit_exceeded_flushes_and_disables_retry( + self, + ): + from google.cloud.spanner_v1._async.snapshot import _restart_on_unavailable + from google.cloud.spanner_v1.types.result_set import PartialResultSet + + chunks = [PartialResultSet(last=False) for _ in range(4)] + pulled_indices = [] + + async def _stream_generator(): + for index, chunk in enumerate(chunks): + pulled_indices.append(index) + yield chunk + raise ServiceUnavailable("transient") + + restart = mock.Mock(side_effect=[_stream_generator()]) + request = mock.Mock() + request.transaction = None + request.resume_token = b"" + session = _Session() + snapshot = self._make_snapshot(session) + + resumable = _restart_on_unavailable( + restart, + request, + metadata=None, + trace_name="span", + session=session, + attributes={}, + transaction=snapshot, + request_id_manager=session._database, + max_buffer_size=2, + ) + + # Pulling the first item buffers 3 chunks (> max_buffer_size=2) and + # flushes them without waiting for chunk 3 or stream completion. + self.assertEqual(await resumable.__anext__(), chunks[0]) + self.assertEqual(pulled_indices, [0, 1, 2]) + self.assertEqual(await resumable.__anext__(), chunks[1]) + self.assertEqual(await resumable.__anext__(), chunks[2]) + + # Once safe_to_retry is False, subsequent tokenless chunks flush immediately. + self.assertEqual(await resumable.__anext__(), chunks[3]) + self.assertEqual(pulled_indices, [0, 1, 2, 3]) + + with self.assertRaisesRegex(ServiceUnavailable, "transient") as context: + await resumable.__anext__() + self.assertTrue(hasattr(context.exception, "request_id")) + self.assertEqual(len(restart.mock_calls), 1) + + async def test_restart_on_unavailable_buffer_limit_exceeded_disables_resumable_internal_error( + self, + ): + from google.cloud.spanner_v1._async.snapshot import _restart_on_unavailable + from google.cloud.spanner_v1.types.result_set import PartialResultSet + + chunk_one = PartialResultSet(last=False) + chunk_two = PartialResultSet(last=False) + stream_one = _MockIterator( + chunk_one, + chunk_two, + fail_after=True, + error=InternalServerError("RST_STREAM"), + ) + restart = mock.Mock(side_effect=[stream_one]) + request = mock.Mock() + request.transaction = None + request.resume_token = b"" + session = _Session() + snapshot = self._make_snapshot(session) + + resumable = _restart_on_unavailable( + restart, + request, + metadata=None, + trace_name="span", + session=session, + attributes={}, + transaction=snapshot, + request_id_manager=session._database, + max_buffer_size=1, + ) + + self.assertEqual(await resumable.__anext__(), chunk_one) + self.assertEqual(await resumable.__anext__(), chunk_two) + with self.assertRaisesRegex(InternalServerError, "RST_STREAM") as context: + await resumable.__anext__() + self.assertTrue(hasattr(context.exception, "request_id")) + self.assertEqual(len(restart.mock_calls), 1) + + async def test_restart_on_unavailable_buffer_limit_recovered_by_newer_resume_token( + self, + ): + from google.cloud.spanner_v1._async.snapshot import _restart_on_unavailable + from google.cloud.spanner_v1.types.result_set import PartialResultSet + + chunk_zero = PartialResultSet(resume_token=b"r1") + chunk_one = PartialResultSet() + chunk_two = PartialResultSet() + chunk_three = PartialResultSet() + chunk_four = PartialResultSet(resume_token=b"r2") + chunk_five_discarded = PartialResultSet() + chunk_five_recovered = PartialResultSet() + chunk_six_last = PartialResultSet(last=True) + + stream_one = _MockIterator( + chunk_zero, + chunk_one, + chunk_two, + chunk_three, + chunk_four, + chunk_five_discarded, + fail_after=True, + error=ServiceUnavailable("transient"), + ) + stream_two = _MockIterator(chunk_five_recovered, chunk_six_last) + restart = mock.Mock(side_effect=[stream_one, stream_two]) + request = mock.Mock() + request.transaction = None + request.resume_token = b"" + session = _Session() + snapshot = self._make_snapshot(session) + + resumable = _restart_on_unavailable( + restart, + request, + metadata=None, + trace_name="span", + session=session, + attributes={}, + transaction=snapshot, + request_id_manager=session._database, + max_buffer_size=1, + ) + received_items = [received async for received in resumable] + + self.assertEqual( + received_items, + [ + chunk_zero, + chunk_one, + chunk_two, + chunk_three, + chunk_four, + chunk_five_recovered, + chunk_six_last, + ], + ) + self.assertEqual(len(restart.mock_calls), 2) + self.assertEqual(request.resume_token, b"r2") + + async def test_restart_on_unavailable_stale_resume_token_does_not_reenable_retry( + self, + ): + from google.cloud.spanner_v1._async.snapshot import _restart_on_unavailable + from google.cloud.spanner_v1.types.result_set import PartialResultSet + + chunk_zero = PartialResultSet(resume_token=b"r1") + chunk_one = PartialResultSet() + chunk_two = PartialResultSet() + chunk_three_stale = PartialResultSet(resume_token=b"r1") + + stream_one = _MockIterator( + chunk_zero, + chunk_one, + chunk_two, + chunk_three_stale, + fail_after=True, + error=ServiceUnavailable("transient"), + ) + restart = mock.Mock(side_effect=[stream_one]) + request = mock.Mock() + request.transaction = None + request.resume_token = b"" + session = _Session() + snapshot = self._make_snapshot(session) + + resumable = _restart_on_unavailable( + restart, + request, + metadata=None, + trace_name="span", + session=session, + attributes={}, + transaction=snapshot, + request_id_manager=session._database, + max_buffer_size=1, + ) + + self.assertEqual(await resumable.__anext__(), chunk_zero) + self.assertEqual(await resumable.__anext__(), chunk_one) + self.assertEqual(await resumable.__anext__(), chunk_two) + self.assertEqual(await resumable.__anext__(), chunk_three_stale) + with self.assertRaisesRegex(ServiceUnavailable, "transient") as context: + await resumable.__anext__() + self.assertTrue(hasattr(context.exception, "request_id")) + self.assertEqual(len(restart.mock_calls), 1) + + async def test_restart_on_unavailable_zero_max_buffer_size(self): + from google.cloud.spanner_v1._async.snapshot import _restart_on_unavailable + from google.cloud.spanner_v1.types.result_set import PartialResultSet + + # Tokenless chunk with max_buffer_size=0 flushes immediately and disables retry. + tokenless_chunk = PartialResultSet() + stream_no_token = _MockIterator( + tokenless_chunk, + fail_after=True, + error=ServiceUnavailable("transient"), + ) + restart_no_token = mock.Mock(side_effect=[stream_no_token]) + request_no_token = mock.Mock() + request_no_token.transaction = None + request_no_token.resume_token = b"" + session = _Session() + snapshot = self._make_snapshot(session) + + resumable_no_token = _restart_on_unavailable( + restart_no_token, + request_no_token, + metadata=None, + trace_name="span", + session=session, + attributes={}, + transaction=snapshot, + request_id_manager=session._database, + max_buffer_size=0, + ) + self.assertEqual(await resumable_no_token.__anext__(), tokenless_chunk) + with self.assertRaisesRegex(ServiceUnavailable, "transient"): + await resumable_no_token.__anext__() + self.assertEqual(len(restart_no_token.mock_calls), 1) + + # Chunk with a new resume_token with max_buffer_size=0 keeps safe_to_retry=True. + token_chunk = PartialResultSet(resume_token=b"r1") + final_chunk = PartialResultSet(last=True) + stream_first = _MockIterator( + token_chunk, + fail_after=True, + error=ServiceUnavailable("transient"), + ) + stream_second = _MockIterator(final_chunk) + restart_with_token = mock.Mock(side_effect=[stream_first, stream_second]) + request_with_token = mock.Mock() + request_with_token.transaction = None + request_with_token.resume_token = b"" + + resumable_with_token = _restart_on_unavailable( + restart_with_token, + request_with_token, + metadata=None, + trace_name="span", + session=session, + attributes={}, + transaction=snapshot, + request_id_manager=session._database, + max_buffer_size=0, + ) + received_items = [received async for received in resumable_with_token] + self.assertEqual(received_items, [token_chunk, final_chunk]) + self.assertEqual(len(restart_with_token.mock_calls), 2) + self.assertEqual(request_with_token.resume_token, b"r1") + + async def test_restart_on_unavailable_buffer_limit_exceeded_completes_cleanly( + self, + ): + from google.cloud.spanner_v1._async.snapshot import _restart_on_unavailable + from google.cloud.spanner_v1.types.result_set import PartialResultSet + + chunks = [PartialResultSet(last=False) for _ in range(4)] + raw = _MockIterator(*chunks) + raw.cancel = mock.Mock() + restart = mock.Mock(return_value=raw) + request = mock.Mock() + request.transaction = None + request.resume_token = b"" + session = _Session() + snapshot = self._make_snapshot(session) + + resumable = _restart_on_unavailable( + restart, + request, + metadata=None, + trace_name="span", + session=session, + attributes={}, + transaction=snapshot, + request_id_manager=session._database, + max_buffer_size=2, + ) + received_items = [received async for received in resumable] + + self.assertEqual(received_items, chunks) + self.assertEqual(len(restart.mock_calls), 1) + raw.cancel.assert_not_called() + + async def test_restart_on_unavailable_new_token_at_limit_and_re_exceed(self): + from google.cloud.spanner_v1._async.snapshot import _restart_on_unavailable + from google.cloud.spanner_v1.types.result_set import PartialResultSet + + # Part A: New resume_token arriving on chunk max_buffer_size + 1 takes + # precedence over buffer-limit flush and keeps safe_to_retry=True. + chunk_zero = PartialResultSet() + chunk_one = PartialResultSet(resume_token=b"r1") + chunk_two = PartialResultSet(last=True) + + stream_one = _MockIterator( + chunk_zero, + chunk_one, + fail_after=True, + error=ServiceUnavailable("transient"), + ) + stream_two = _MockIterator(chunk_two) + restart = mock.Mock(side_effect=[stream_one, stream_two]) + request = mock.Mock() + request.transaction = None + request.resume_token = b"" + session = _Session() + snapshot = self._make_snapshot(session) + + resumable = _restart_on_unavailable( + restart, + request, + metadata=None, + trace_name="span", + session=session, + attributes={}, + transaction=snapshot, + request_id_manager=session._database, + max_buffer_size=1, + ) + received_items = [received async for received in resumable] + self.assertEqual(received_items, [chunk_zero, chunk_one, chunk_two]) + self.assertEqual(len(restart.mock_calls), 2) + self.assertEqual(request.resume_token, b"r1") + + # Part B: Transition True -> False -> True (newer token) -> False (re-exceed). + re_exceed_chunks = [ + PartialResultSet(), + PartialResultSet(), + PartialResultSet(resume_token=b"r2"), + PartialResultSet(), + PartialResultSet(), + ] + stream_re_exceed = _MockIterator( + *re_exceed_chunks, + fail_after=True, + error=ServiceUnavailable("second_failure"), + ) + restart_re_exceed = mock.Mock(side_effect=[stream_re_exceed]) + request_re_exceed = mock.Mock() + request_re_exceed.transaction = None + request_re_exceed.resume_token = b"" + + resumable_re_exceed = _restart_on_unavailable( + restart_re_exceed, + request_re_exceed, + metadata=None, + trace_name="span", + session=session, + attributes={}, + transaction=snapshot, + request_id_manager=session._database, + max_buffer_size=1, + ) + collected_items = [] + with self.assertRaisesRegex(ServiceUnavailable, "second_failure"): + async for received in resumable_re_exceed: + collected_items.append(received) + self.assertEqual(collected_items, re_exceed_chunks) + self.assertEqual(len(restart_re_exceed.mock_calls), 1) + + async def test_restart_on_unavailable_default_max_buffer_size_streamed_result_set( + self, + ): + from google.protobuf.struct_pb2 import Value + + from google.cloud.spanner_v1._async.snapshot import ( + _MAX_BUFFERED_CHUNKS, + _restart_on_unavailable, + ) + from google.cloud.spanner_v1._async.streamed import StreamedResultSet + from google.cloud.spanner_v1.types.result_set import ( + PartialResultSet, + ResultSetMetadata, + ) + from google.cloud.spanner_v1.types.type import StructType, Type, TypeCode + + self.assertEqual(_MAX_BUFFERED_CHUNKS, 512) + fields = [StructType.Field(name="id", type_=Type(code=TypeCode.INT64))] + metadata_pb = ResultSetMetadata(row_type=StructType(fields=fields)) + + def _make_chunk(index: int, include_metadata: bool = False) -> PartialResultSet: + chunk = PartialResultSet( + metadata=metadata_pb if include_metadata else None, + ) + chunk.values.append(Value(string_value=str(index))) + return chunk + + # Exactly 512 tokenless chunks remain within the threshold and retry cleanly. + within_limit_chunks = [ + _make_chunk(index, include_metadata=(index == 0)) + for index in range(_MAX_BUFFERED_CHUNKS) + ] + recovered_chunk = _make_chunk(_MAX_BUFFERED_CHUNKS, include_metadata=True) + recovered_chunk.last = True + stream_within_one = _MockIterator( + *within_limit_chunks, + fail_after=True, + error=ServiceUnavailable("transient"), + ) + stream_within_two = _MockIterator(recovered_chunk) + restart_within = mock.Mock(side_effect=[stream_within_one, stream_within_two]) + request_within = mock.Mock() + request_within.transaction = None + request_within.resume_token = b"" + session = _Session() + snapshot = self._make_snapshot(session) + + resumable_within = _restart_on_unavailable( + restart_within, + request_within, + metadata=None, + trace_name="span", + session=session, + attributes={}, + transaction=snapshot, + request_id_manager=session._database, + ) + result_set_within = StreamedResultSet(resumable_within) + rows_within = [row async for row in result_set_within] + self.assertEqual(rows_within, [[_MAX_BUFFERED_CHUNKS]]) + self.assertEqual(len(restart_within.mock_calls), 2) + + # 513 tokenless chunks exceed the default 512 threshold, flush downstream, + # and propagate the subsequent ServiceUnavailable without retrying. + over_limit_chunks = [ + _make_chunk(index, include_metadata=(index == 0)) + for index in range(_MAX_BUFFERED_CHUNKS + 1) + ] + stream_over = _MockIterator( + *over_limit_chunks, + fail_after=True, + error=ServiceUnavailable("exceeded"), + ) + restart_over = mock.Mock(side_effect=[stream_over]) + request_over = mock.Mock() + request_over.transaction = None + request_over.resume_token = b"" + + resumable_over = _restart_on_unavailable( + restart_over, + request_over, + metadata=None, + trace_name="span", + session=session, + attributes={}, + transaction=snapshot, + request_id_manager=session._database, + ) + result_set_over = StreamedResultSet(resumable_over) + collected_rows = [] + with self.assertRaisesRegex(ServiceUnavailable, "exceeded"): + async for row in result_set_over: + collected_rows.append(row) + self.assertEqual( + collected_rows, [[index] for index in range(_MAX_BUFFERED_CHUNKS + 1)] + ) + self.assertEqual(len(restart_over.mock_calls), 1) + async def test_restart_on_unavailable_metrics_tracer_lifecycle(self): from google.cloud.spanner_v1._async.snapshot import _restart_on_unavailable from google.cloud.spanner_v1.metrics.metrics_tracer_factory import ( diff --git a/packages/google-cloud-spanner/tests/unit/test_snapshot.py b/packages/google-cloud-spanner/tests/unit/test_snapshot.py index b236ec2d8424..a754d7dc81fa 100644 --- a/packages/google-cloud-spanner/tests/unit/test_snapshot.py +++ b/packages/google-cloud-spanner/tests/unit/test_snapshot.py @@ -146,6 +146,7 @@ def _call_fut( session=None, attributes=None, metadata=None, + max_buffer_size=None, ): from google.cloud.spanner_v1.snapshot import _restart_on_unavailable @@ -158,6 +159,7 @@ def _call_fut( attributes, transaction=derived, request_id_manager=None if not session else session._database, + max_buffer_size=max_buffer_size, ) def _make_item(self, value, resume_token=b"", metadata=None, last=False): @@ -434,6 +436,439 @@ def test_restart_on_unavailable_retry_before_last(self): mock_drain.assert_called_once_with(stream_two) stream_two.cancel.assert_not_called() + def test_restart_on_unavailable_negative_max_buffer_size(self): + restart = mock.Mock() + request = mock.Mock(test="test", spec=["test", "resume_token"]) + database = _Database() + database.spanner_api = build_spanner_api() + session = _Session(database) + derived = _build_snapshot_derived(session) + + resumable = self._call_fut( + derived, restart, request, session=session, max_buffer_size=-1 + ) + with self.assertRaisesRegex(ValueError, "max_buffer_size must be non-negative"): + list(resumable) + restart.assert_not_called() + + def test_restart_on_unavailable_buffer_limit_within_threshold_retries(self): + from google.api_core.exceptions import ServiceUnavailable + + from google.cloud.spanner_v1.types.result_set import PartialResultSet + + discarded_chunk_one = PartialResultSet(last=False) + discarded_chunk_two = PartialResultSet(last=False) + recovered_chunk = PartialResultSet(last=True) + + stream_one = _MockIterator( + discarded_chunk_one, + discarded_chunk_two, + fail_after=True, + error=ServiceUnavailable("transient"), + ) + stream_two = _MockIterator(recovered_chunk) + restart = mock.Mock(side_effect=[stream_one, stream_two]) + request = mock.Mock(test="test", spec=["test", "resume_token"]) + database = _Database() + database.spanner_api = build_spanner_api() + session = _Session(database) + derived = _build_snapshot_derived(session) + + resumable = self._call_fut( + derived, restart, request, session=session, max_buffer_size=2 + ) + received_items = list(resumable) + + self.assertEqual(received_items, [recovered_chunk]) + self.assertEqual(len(restart.mock_calls), 2) + self.assertEqual(request.resume_token, b"") + + def test_restart_on_unavailable_buffer_limit_exceeded_flushes_and_disables_retry( + self, + ): + from google.api_core.exceptions import ServiceUnavailable + + from google.cloud.spanner_v1.types.result_set import PartialResultSet + + chunks = [PartialResultSet(last=False) for _ in range(4)] + pulled_indices = [] + + def _stream_generator(): + for index, chunk in enumerate(chunks): + pulled_indices.append(index) + yield chunk + raise ServiceUnavailable("transient") + + restart = mock.Mock(side_effect=[_stream_generator()]) + request = mock.Mock(test="test", spec=["test", "resume_token"]) + database = _Database() + database.spanner_api = build_spanner_api() + session = _Session(database) + derived = _build_snapshot_derived(session) + + resumable = self._call_fut( + derived, restart, request, session=session, max_buffer_size=2 + ) + + # Pulling the first item buffers 3 chunks (> max_buffer_size=2) and + # flushes them without waiting for chunk 3 or stream completion. + self.assertEqual(next(resumable), chunks[0]) + self.assertEqual(pulled_indices, [0, 1, 2]) + self.assertEqual(next(resumable), chunks[1]) + self.assertEqual(next(resumable), chunks[2]) + + # Once safe_to_retry is False, subsequent tokenless chunks flush immediately. + self.assertEqual(next(resumable), chunks[3]) + self.assertEqual(pulled_indices, [0, 1, 2, 3]) + + with self.assertRaisesRegex(ServiceUnavailable, "transient") as context: + next(resumable) + self.assertTrue(hasattr(context.exception, "request_id")) + self.assertEqual(len(restart.mock_calls), 1) + + def test_restart_on_unavailable_buffer_limit_exceeded_disables_resumable_internal_error( + self, + ): + from google.api_core.exceptions import InternalServerError + + from google.cloud.spanner_v1.types.result_set import PartialResultSet + + chunk_one = PartialResultSet(last=False) + chunk_two = PartialResultSet(last=False) + stream_one = _MockIterator( + chunk_one, + chunk_two, + fail_after=True, + error=InternalServerError("RST_STREAM"), + ) + restart = mock.Mock(side_effect=[stream_one]) + request = mock.Mock(test="test", spec=["test", "resume_token"]) + database = _Database() + database.spanner_api = build_spanner_api() + session = _Session(database) + derived = _build_snapshot_derived(session) + + resumable = self._call_fut( + derived, restart, request, session=session, max_buffer_size=1 + ) + + self.assertEqual(next(resumable), chunk_one) + self.assertEqual(next(resumable), chunk_two) + with self.assertRaisesRegex(InternalServerError, "RST_STREAM") as context: + next(resumable) + self.assertTrue(hasattr(context.exception, "request_id")) + self.assertEqual(len(restart.mock_calls), 1) + + def test_restart_on_unavailable_buffer_limit_recovered_by_newer_resume_token( + self, + ): + from google.api_core.exceptions import ServiceUnavailable + + from google.cloud.spanner_v1.types.result_set import PartialResultSet + + chunk_zero = PartialResultSet(resume_token=b"r1") + chunk_one = PartialResultSet() + chunk_two = PartialResultSet() + chunk_three = PartialResultSet() + chunk_four = PartialResultSet(resume_token=b"r2") + chunk_five_discarded = PartialResultSet() + chunk_five_recovered = PartialResultSet() + chunk_six_last = PartialResultSet(last=True) + + stream_one = _MockIterator( + chunk_zero, + chunk_one, + chunk_two, + chunk_three, + chunk_four, + chunk_five_discarded, + fail_after=True, + error=ServiceUnavailable("transient"), + ) + stream_two = _MockIterator(chunk_five_recovered, chunk_six_last) + restart = mock.Mock(side_effect=[stream_one, stream_two]) + request = mock.Mock(test="test", spec=["test", "resume_token"]) + database = _Database() + database.spanner_api = build_spanner_api() + session = _Session(database) + derived = _build_snapshot_derived(session) + + resumable = self._call_fut( + derived, restart, request, session=session, max_buffer_size=1 + ) + received_items = list(resumable) + + self.assertEqual( + received_items, + [ + chunk_zero, + chunk_one, + chunk_two, + chunk_three, + chunk_four, + chunk_five_recovered, + chunk_six_last, + ], + ) + self.assertEqual(len(restart.mock_calls), 2) + self.assertEqual(request.resume_token, b"r2") + + def test_restart_on_unavailable_stale_resume_token_does_not_reenable_retry(self): + from google.api_core.exceptions import ServiceUnavailable + + from google.cloud.spanner_v1.types.result_set import PartialResultSet + + chunk_zero = PartialResultSet(resume_token=b"r1") + chunk_one = PartialResultSet() + chunk_two = PartialResultSet() + chunk_three_stale = PartialResultSet(resume_token=b"r1") + + stream_one = _MockIterator( + chunk_zero, + chunk_one, + chunk_two, + chunk_three_stale, + fail_after=True, + error=ServiceUnavailable("transient"), + ) + restart = mock.Mock(side_effect=[stream_one]) + request = mock.Mock(test="test", spec=["test", "resume_token"]) + database = _Database() + database.spanner_api = build_spanner_api() + session = _Session(database) + derived = _build_snapshot_derived(session) + + resumable = self._call_fut( + derived, restart, request, session=session, max_buffer_size=1 + ) + + self.assertEqual(next(resumable), chunk_zero) + self.assertEqual(next(resumable), chunk_one) + self.assertEqual(next(resumable), chunk_two) + self.assertEqual(next(resumable), chunk_three_stale) + with self.assertRaisesRegex(ServiceUnavailable, "transient") as context: + next(resumable) + self.assertTrue(hasattr(context.exception, "request_id")) + self.assertEqual(len(restart.mock_calls), 1) + + def test_restart_on_unavailable_zero_max_buffer_size(self): + from google.api_core.exceptions import ServiceUnavailable + + from google.cloud.spanner_v1.types.result_set import PartialResultSet + + # Tokenless chunk with max_buffer_size=0 flushes immediately and disables retry. + tokenless_chunk = PartialResultSet() + stream_no_token = _MockIterator( + tokenless_chunk, + fail_after=True, + error=ServiceUnavailable("transient"), + ) + restart_no_token = mock.Mock(side_effect=[stream_no_token]) + request_no_token = mock.Mock(test="test", spec=["test", "resume_token"]) + database = _Database() + database.spanner_api = build_spanner_api() + session = _Session(database) + derived = _build_snapshot_derived(session) + + resumable_no_token = self._call_fut( + derived, + restart_no_token, + request_no_token, + session=session, + max_buffer_size=0, + ) + self.assertEqual(next(resumable_no_token), tokenless_chunk) + with self.assertRaisesRegex(ServiceUnavailable, "transient"): + next(resumable_no_token) + self.assertEqual(len(restart_no_token.mock_calls), 1) + + # Chunk with a new resume_token with max_buffer_size=0 keeps safe_to_retry=True. + token_chunk = PartialResultSet(resume_token=b"r1") + final_chunk = PartialResultSet(last=True) + stream_first = _MockIterator( + token_chunk, + fail_after=True, + error=ServiceUnavailable("transient"), + ) + stream_second = _MockIterator(final_chunk) + restart_with_token = mock.Mock(side_effect=[stream_first, stream_second]) + request_with_token = mock.Mock(test="test", spec=["test", "resume_token"]) + + resumable_with_token = self._call_fut( + derived, + restart_with_token, + request_with_token, + session=session, + max_buffer_size=0, + ) + received_items = list(resumable_with_token) + self.assertEqual(received_items, [token_chunk, final_chunk]) + self.assertEqual(len(restart_with_token.mock_calls), 2) + self.assertEqual(request_with_token.resume_token, b"r1") + + def test_restart_on_unavailable_buffer_limit_exceeded_completes_cleanly(self): + from google.cloud.spanner_v1.types.result_set import PartialResultSet + + chunks = [PartialResultSet(last=False) for _ in range(4)] + raw = _MockIterator(*chunks) + raw.cancel = mock.Mock() + restart = mock.Mock(return_value=raw) + request = mock.Mock(test="test", spec=["test", "resume_token"]) + database = _Database() + database.spanner_api = build_spanner_api() + session = _Session(database) + derived = _build_snapshot_derived(session) + + resumable = self._call_fut( + derived, restart, request, session=session, max_buffer_size=2 + ) + received_items = list(resumable) + + self.assertEqual(received_items, chunks) + self.assertEqual(len(restart.mock_calls), 1) + raw.cancel.assert_not_called() + + def test_restart_on_unavailable_new_token_at_limit_and_re_exceed(self): + from google.api_core.exceptions import ServiceUnavailable + + from google.cloud.spanner_v1.types.result_set import PartialResultSet + + # Part A: New resume_token arriving on chunk max_buffer_size + 1 takes + # precedence over buffer-limit flush and keeps safe_to_retry=True. + chunk_zero = PartialResultSet() + chunk_one = PartialResultSet(resume_token=b"r1") + chunk_two = PartialResultSet(last=True) + + stream_one = _MockIterator( + chunk_zero, + chunk_one, + fail_after=True, + error=ServiceUnavailable("transient"), + ) + stream_two = _MockIterator(chunk_two) + restart = mock.Mock(side_effect=[stream_one, stream_two]) + request = mock.Mock(test="test", spec=["test", "resume_token"]) + database = _Database() + database.spanner_api = build_spanner_api() + session = _Session(database) + derived = _build_snapshot_derived(session) + + resumable = self._call_fut( + derived, restart, request, session=session, max_buffer_size=1 + ) + received_items = list(resumable) + self.assertEqual(received_items, [chunk_zero, chunk_one, chunk_two]) + self.assertEqual(len(restart.mock_calls), 2) + self.assertEqual(request.resume_token, b"r1") + + # Part B: Transition True -> False -> True (newer token) -> False (re-exceed). + re_exceed_chunks = [ + PartialResultSet(), + PartialResultSet(), + PartialResultSet(resume_token=b"r2"), + PartialResultSet(), + PartialResultSet(), + ] + stream_re_exceed = _MockIterator( + *re_exceed_chunks, + fail_after=True, + error=ServiceUnavailable("second_failure"), + ) + restart_re_exceed = mock.Mock(side_effect=[stream_re_exceed]) + request_re_exceed = mock.Mock(test="test", spec=["test", "resume_token"]) + + resumable_re_exceed = self._call_fut( + derived, + restart_re_exceed, + request_re_exceed, + session=session, + max_buffer_size=1, + ) + collected_items = [] + with self.assertRaisesRegex(ServiceUnavailable, "second_failure"): + for received in resumable_re_exceed: + collected_items.append(received) + self.assertEqual(collected_items, re_exceed_chunks) + self.assertEqual(len(restart_re_exceed.mock_calls), 1) + + def test_restart_on_unavailable_default_max_buffer_size_streamed_result_set(self): + from google.api_core.exceptions import ServiceUnavailable + from google.protobuf.struct_pb2 import Value + + from google.cloud.spanner_v1.snapshot import _MAX_BUFFERED_CHUNKS + from google.cloud.spanner_v1.streamed import StreamedResultSet + from google.cloud.spanner_v1.types.result_set import ( + PartialResultSet, + ResultSetMetadata, + ) + from google.cloud.spanner_v1.types.type import StructType, Type, TypeCode + + self.assertEqual(_MAX_BUFFERED_CHUNKS, 512) + fields = [StructType.Field(name="id", type_=Type(code=TypeCode.INT64))] + metadata_pb = ResultSetMetadata(row_type=StructType(fields=fields)) + + def _make_chunk(index: int, include_metadata: bool = False) -> PartialResultSet: + chunk = PartialResultSet( + metadata=metadata_pb if include_metadata else None, + ) + chunk.values.append(Value(string_value=str(index))) + return chunk + + # Exactly 512 tokenless chunks remain within the threshold and retry cleanly. + within_limit_chunks = [ + _make_chunk(index, include_metadata=(index == 0)) + for index in range(_MAX_BUFFERED_CHUNKS) + ] + recovered_chunk = _make_chunk(_MAX_BUFFERED_CHUNKS, include_metadata=True) + recovered_chunk.last = True + stream_within_one = _MockIterator( + *within_limit_chunks, + fail_after=True, + error=ServiceUnavailable("transient"), + ) + stream_within_two = _MockIterator(recovered_chunk) + restart_within = mock.Mock(side_effect=[stream_within_one, stream_within_two]) + request_within = mock.Mock(test="test", spec=["test", "resume_token"]) + database = _Database() + database.spanner_api = build_spanner_api() + session = _Session(database) + derived = _build_snapshot_derived(session) + + resumable_within = self._call_fut( + derived, restart_within, request_within, session=session + ) + result_set_within = StreamedResultSet(resumable_within) + rows_within = list(result_set_within) + self.assertEqual(rows_within, [[_MAX_BUFFERED_CHUNKS]]) + self.assertEqual(len(restart_within.mock_calls), 2) + + # 513 tokenless chunks exceed the default 512 threshold, flush downstream, + # and propagate the subsequent ServiceUnavailable without retrying. + over_limit_chunks = [ + _make_chunk(index, include_metadata=(index == 0)) + for index in range(_MAX_BUFFERED_CHUNKS + 1) + ] + stream_over = _MockIterator( + *over_limit_chunks, + fail_after=True, + error=ServiceUnavailable("exceeded"), + ) + restart_over = mock.Mock(side_effect=[stream_over]) + request_over = mock.Mock(test="test", spec=["test", "resume_token"]) + + resumable_over = self._call_fut( + derived, restart_over, request_over, session=session + ) + result_set_over = StreamedResultSet(resumable_over) + collected_rows = [] + with self.assertRaisesRegex(ServiceUnavailable, "exceeded"): + for row in result_set_over: + collected_rows.append(row) + self.assertEqual( + collected_rows, [[index] for index in range(_MAX_BUFFERED_CHUNKS + 1)] + ) + self.assertEqual(len(restart_over.mock_calls), 1) + def test_iteration_w_raw_w_resume_token(self): ITEMS = ( self._make_item(0),