Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,12 @@
_TRANSACTION_BEGIN_TIMEOUT_SECONDS = 30.0


# Maximum number of PartialResultSet chunks to buffer while waiting for a
# resume_token before flushing downstream and disabling retries until a newer
# resume_token arrives.
_MAX_BUFFERED_CHUNKS = 512


@CrossSync.convert
async def _restart_on_unavailable(
method,
Expand All @@ -89,6 +95,7 @@ async def _restart_on_unavailable(
observability_options=None,
request_id_manager=None,
resource_info=None,
max_buffer_size: Optional[int] = None,
):
"""Restart iteration after :exc:`.ServiceUnavailable`.

Expand All @@ -104,10 +111,22 @@ async def _restart_on_unavailable(
:type transaction_selector: :class:`transaction_pb2.TransactionSelector`
:param transaction_selector: Transaction selector object to be used in request if transaction is not passed,
if both transaction_selector and transaction are passed, then transaction is given priority.

:type max_buffer_size: int
:param max_buffer_size:
(Optional) Maximum number of ``PartialResultSet`` items to buffer
while waiting for a ``resume_token``. Defaults to
``_MAX_BUFFERED_CHUNKS``.
"""

if max_buffer_size is None:
max_buffer_size = _MAX_BUFFERED_CHUNKS
elif max_buffer_size < 0:
raise ValueError("max_buffer_size must be non-negative")

resume_token: bytes = b""
item_buffer: List[PartialResultSet] = []
safe_to_retry: bool = True

if transaction is not None:
transaction_selector = transaction._build_transaction_selector_pb()
Expand Down Expand Up @@ -199,35 +218,32 @@ async def _restart_on_unavailable(
iterator = None
break

if item.resume_token:
if item.resume_token and item.resume_token != resume_token:
resume_token = item.resume_token
safe_to_retry = True
break

if not safe_to_retry or len(item_buffer) > max_buffer_size:
# Flush without a resume_token; retries remain unsafe
# until a newer resume_token supersedes the flushed items.
safe_to_retry = False
break
else:
stream_finished = True

except ServiceUnavailable:
del item_buffer[:]
request.resume_token = resume_token
if transaction is not None:
transaction_selector = transaction._build_transaction_selector_pb()
request.transaction = transaction_selector
attempt += 1
iterator = None
continue

except InternalServerError as exc:
resumable_error = any(
except (ServiceUnavailable, InternalServerError) as exc:
is_retryable = isinstance(exc, ServiceUnavailable) or any(
resumable_message in exc.message
for resumable_message in _STREAM_RESUMPTION_INTERNAL_ERROR_MESSAGES
)
if not resumable_error:
if not safe_to_retry or not is_retryable:
raise _augment_error_with_request_id(exc, current_request_id)
del item_buffer[:]
request.resume_token = resume_token
if transaction is not None:
transaction_selector = transaction._build_transaction_selector_pb()
attempt += 1
request.transaction = transaction_selector
attempt += 1
iterator = None
continue

Expand Down
38 changes: 23 additions & 15 deletions packages/google-cloud-spanner/google/cloud/spanner_v1/snapshot.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@
"Received unexpected EOS on DATA frame from server",
)
_TRANSACTION_BEGIN_TIMEOUT_SECONDS = 30.0
_MAX_BUFFERED_CHUNKS = 512


def _restart_on_unavailable(
Expand All @@ -88,6 +89,7 @@ def _restart_on_unavailable(
observability_options=None,
request_id_manager=None,
resource_info=None,
max_buffer_size: Optional[int] = None,
):
"""Restart iteration after :exc:`.ServiceUnavailable`.

Expand All @@ -102,9 +104,20 @@ def _restart_on_unavailable(

:type transaction_selector: :class:`transaction_pb2.TransactionSelector`
:param transaction_selector: Transaction selector object to be used in request if transaction is not passed,
if both transaction_selector and transaction are passed, then transaction is given priority."""
if both transaction_selector and transaction are passed, then transaction is given priority.

:type max_buffer_size: int
:param max_buffer_size:
(Optional) Maximum number of ``PartialResultSet`` items to buffer
while waiting for a ``resume_token``. Defaults to
``_MAX_BUFFERED_CHUNKS``."""
if max_buffer_size is None:
max_buffer_size = _MAX_BUFFERED_CHUNKS
elif max_buffer_size < 0:
raise ValueError("max_buffer_size must be non-negative")
resume_token: bytes = b""
item_buffer: List[PartialResultSet] = []
safe_to_retry: bool = True
if transaction is not None:
transaction_selector = transaction._build_transaction_selector_pb()
elif transaction_selector is None:
Expand Down Expand Up @@ -176,35 +189,30 @@ def _restart_on_unavailable(
_drain_stream(iterator)
iterator = None
break
if item.resume_token:
if item.resume_token and item.resume_token != resume_token:
resume_token = item.resume_token
safe_to_retry = True
break
if not safe_to_retry or len(item_buffer) > max_buffer_size:
safe_to_retry = False
break
Comment thread
olavloite marked this conversation as resolved.
Resolved
else:
stream_finished = True
except ServiceUnavailable:
del item_buffer[:]
request.resume_token = resume_token
if transaction is not None:
transaction_selector = transaction._build_transaction_selector_pb()
request.transaction = transaction_selector
attempt += 1
iterator = None
continue
except InternalServerError as exc:
resumable_error = any(
except (ServiceUnavailable, InternalServerError) as exc:
is_retryable = isinstance(exc, ServiceUnavailable) or any(
(
resumable_message in exc.message
for resumable_message in _STREAM_RESUMPTION_INTERNAL_ERROR_MESSAGES
)
)
Comment thread
olavloite marked this conversation as resolved.
Resolved
if not resumable_error:
if not safe_to_retry or not is_retryable:
raise _augment_error_with_request_id(exc, current_request_id)
del item_buffer[:]
request.resume_token = resume_token
if transaction is not None:
transaction_selector = transaction._build_transaction_selector_pb()
attempt += 1
request.transaction = transaction_selector
attempt += 1
iterator = None
continue
except Exception as exc:
Expand Down
190 changes: 190 additions & 0 deletions packages/google-cloud-spanner/tests/mockserver_tests/test_basics.py
Original file line number Diff line number Diff line change
Expand Up @@ -676,6 +676,97 @@ def test_partial_result_set_without_resume_tokens_and_explicit_chunked_value(sel
[b"", b"mid_tok"], [request.resume_token for request in execute_requests]
)

def test_partial_result_set_buffer_limit_safe_to_retry(self):
from google.cloud.spanner_v1.snapshot import _MAX_BUFFERED_CHUNKS

# 1. Exactly 512 tokenless chunks stay within the threshold and retry cleanly.
sql_within = "select id from buffer_limit_within"
within_rows = [(str(index),) for index in range(_MAX_BUFFERED_CHUNKS)]
add_multi_column_result(
sql_within,
[("id", TypeCode.INT64)],
within_rows,
chunk_config=PartialResultSetConfig(
max_values_per_chunk=1, emit_resume_tokens=False
),
)
add_execute_streaming_sql_error(
sql_within, _MAX_BUFFERED_CHUNKS, unavailable_status()
)
with self.database.snapshot() as snapshot:
rows = list(snapshot.execute_sql(sql_within))
self.assertEqual([[index] for index in range(_MAX_BUFFERED_CHUNKS)], rows)
execute_requests = _requests_of_type(self.spanner_service, ExecuteSqlRequest)
self.assertEqual(
[b"", b""], [request.resume_token for request in execute_requests]
)

# 2. 513 tokenless chunks exceed the threshold, flush downstream, and fail
# without retrying on subsequent UNAVAILABLE.
sql_exceeded = "select id from buffer_limit_exceeded"
exceeded_rows = [(str(index),) for index in range(_MAX_BUFFERED_CHUNKS + 1)]
add_multi_column_result(
sql_exceeded,
[("id", TypeCode.INT64)],
exceeded_rows,
chunk_config=PartialResultSetConfig(
max_values_per_chunk=1, emit_resume_tokens=False
),
)
self.spanner_service.clear_requests()
add_execute_streaming_sql_error(
sql_exceeded, _MAX_BUFFERED_CHUNKS + 1, unavailable_status()
)
collected_rows = []
with self.database.snapshot() as snapshot:
with self.assertRaisesRegex(
exceptions.ServiceUnavailable, "Service unavailable."
):
for row in snapshot.execute_sql(sql_exceeded):
collected_rows.append(row)
self.assertEqual(
[[index] for index in range(_MAX_BUFFERED_CHUNKS + 1)], collected_rows
)
execute_requests = _requests_of_type(self.spanner_service, ExecuteSqlRequest)
self.assertEqual([b""], [request.resume_token for request in execute_requests])

# 3. Exceeding the threshold followed by a newer resume_token re-enables retry.
sql_recovered = "select id from buffer_limit_recovered"
partial_specs = [
{"values": [str(index)]} for index in range(_MAX_BUFFERED_CHUNKS + 1)
]
partial_specs.append(
{
"values": [str(_MAX_BUFFERED_CHUNKS + 1)],
"resume_token": b"tok_recover",
}
)
partial_specs.append({"values": [str(_MAX_BUFFERED_CHUNKS + 2)]})
partial_specs.append(
{
"values": [str(_MAX_BUFFERED_CHUNKS + 3)],
"last": True,
}
)
add_execute_streaming_sql_results(
sql_recovered,
_make_partial_result_sets([("id", TypeCode.INT64)], partial_specs),
)
self.spanner_service.clear_requests()
add_execute_streaming_sql_error(
sql_recovered, _MAX_BUFFERED_CHUNKS + 3, unavailable_status()
)
with self.database.snapshot() as snapshot:
rows = list(snapshot.execute_sql(sql_recovered))
self.assertEqual(
[[index] for index in range(_MAX_BUFFERED_CHUNKS + 4)], rows
)
execute_requests = _requests_of_type(self.spanner_service, ExecuteSqlRequest)
self.assertEqual(
[b"", b"tok_recover"],
[request.resume_token for request in execute_requests],
)

def test_partial_result_set_config_validation(self):
# Verify PartialResultSetConfig rejects non-positive chunking thresholds.
with self.assertRaisesRegex(
Expand Down Expand Up @@ -1199,6 +1290,105 @@ async def test_async_execute_streaming_sql_last_flag_trailing_error_drained(self
rows = [row async for row in results]
self.assertEqual([[10], [20]], rows)

async def test_async_partial_result_set_buffer_limit_safe_to_retry(self):
from google.cloud.spanner_v1._async.snapshot import _MAX_BUFFERED_CHUNKS

# 1. Exactly 512 tokenless chunks stay within the threshold and retry cleanly.
sql_within = "select id from async_buffer_limit_within"
within_rows = [(str(index),) for index in range(_MAX_BUFFERED_CHUNKS)]
add_multi_column_result(
sql_within,
[("id", TypeCode.INT64)],
within_rows,
chunk_config=PartialResultSetConfig(
max_values_per_chunk=1, emit_resume_tokens=False
),
)
add_execute_streaming_sql_error(
sql_within, _MAX_BUFFERED_CHUNKS, unavailable_status()
)
async with self._async_database() as database:
async with database.snapshot() as snapshot:
results = await snapshot.execute_sql(sql_within)
rows = [row async for row in results]
self.assertEqual(
[[index] for index in range(_MAX_BUFFERED_CHUNKS)], rows
)
execute_requests = _requests_of_type(self.spanner_service, ExecuteSqlRequest)
self.assertEqual(
[b"", b""], [request.resume_token for request in execute_requests]
)

# 2. 513 tokenless chunks exceed the threshold, flush downstream, and fail
# without retrying on subsequent UNAVAILABLE.
sql_exceeded = "select id from async_buffer_limit_exceeded"
exceeded_rows = [(str(index),) for index in range(_MAX_BUFFERED_CHUNKS + 1)]
add_multi_column_result(
sql_exceeded,
[("id", TypeCode.INT64)],
exceeded_rows,
chunk_config=PartialResultSetConfig(
max_values_per_chunk=1, emit_resume_tokens=False
),
)
self.spanner_service.clear_requests()
add_execute_streaming_sql_error(
sql_exceeded, _MAX_BUFFERED_CHUNKS + 1, unavailable_status()
)
collected_rows = []
async with self._async_database() as database:
async with database.snapshot() as snapshot:
results = await snapshot.execute_sql(sql_exceeded)
with self.assertRaisesRegex(
exceptions.ServiceUnavailable, "Service unavailable."
):
async for row in results:
collected_rows.append(row)
self.assertEqual(
[[index] for index in range(_MAX_BUFFERED_CHUNKS + 1)], collected_rows
)
execute_requests = _requests_of_type(self.spanner_service, ExecuteSqlRequest)
self.assertEqual([b""], [request.resume_token for request in execute_requests])

# 3. Exceeding the threshold followed by a newer resume_token re-enables retry.
sql_recovered = "select id from async_buffer_limit_recovered"
partial_specs = [
{"values": [str(index)]} for index in range(_MAX_BUFFERED_CHUNKS + 1)
]
partial_specs.append(
{
"values": [str(_MAX_BUFFERED_CHUNKS + 1)],
"resume_token": b"tok_recover",
}
)
partial_specs.append({"values": [str(_MAX_BUFFERED_CHUNKS + 2)]})
partial_specs.append(
{
"values": [str(_MAX_BUFFERED_CHUNKS + 3)],
"last": True,
}
)
add_execute_streaming_sql_results(
sql_recovered,
_make_partial_result_sets([("id", TypeCode.INT64)], partial_specs),
)
self.spanner_service.clear_requests()
add_execute_streaming_sql_error(
sql_recovered, _MAX_BUFFERED_CHUNKS + 3, unavailable_status()
)
async with self._async_database() as database:
async with database.snapshot() as snapshot:
results = await snapshot.execute_sql(sql_recovered)
rows = [row async for row in results]
self.assertEqual(
[[index] for index in range(_MAX_BUFFERED_CHUNKS + 4)], rows
)
execute_requests = _requests_of_type(self.spanner_service, ExecuteSqlRequest)
self.assertEqual(
[b"", b"tok_recover"],
[request.resume_token for request in execute_requests],
)


def _execute_query(transaction: Transaction, sql: str):
rows = transaction.execute_sql(sql, last_statement=True)
Expand Down
Loading
Loading