From 028ad000c78645299e709e38a0a43ea11ae85be3 Mon Sep 17 00:00:00 2001 From: zhixiangli Date: Fri, 20 Nov 2026 14:17:20 +0000 Subject: [PATCH 1/3] fix(storage): reopen the MRD stream after a cancelled reopen --- .../storage/asyncio/_stream_multiplexer.py | 8 ++ .../test_async_multi_range_downloader.py | 96 +++++++++++++++++++ .../unit/asyncio/test_stream_multiplexer.py | 45 +++++++++ 3 files changed, 149 insertions(+) diff --git a/packages/google-cloud-storage/google/cloud/storage/asyncio/_stream_multiplexer.py b/packages/google-cloud-storage/google/cloud/storage/asyncio/_stream_multiplexer.py index 6251266b7240..691833d8e95a 100644 --- a/packages/google-cloud-storage/google/cloud/storage/asyncio/_stream_multiplexer.py +++ b/packages/google-cloud-storage/google/cloud/storage/asyncio/_stream_multiplexer.py @@ -19,6 +19,7 @@ from typing import Awaitable, Callable, Dict, Optional, Set import grpc +from google.api_core import exceptions from google.cloud import _storage_v2 from google.cloud.storage.asyncio.async_read_object_stream import ( @@ -181,6 +182,13 @@ async def _recv_loop(self) -> None: self._put_error_nowait(queue, error) async def send(self, request: _storage_v2.BidiReadObjectRequest) -> int: + if not self._stream.is_stream_open: + # A reopen cancelled after closing the old stream leaves that + # closed stream on the multiplexer at the same generation; raise a + # retryable error so the caller's retry reopens it. + raise exceptions.ServiceUnavailable( + "Bidi read stream is not open; it will be reopened" + ) self._ensure_recv_loop() await self._stream.send(request) return self._stream_generation diff --git a/packages/google-cloud-storage/tests/unit/asyncio/test_async_multi_range_downloader.py b/packages/google-cloud-storage/tests/unit/asyncio/test_async_multi_range_downloader.py index 74befb8eda2e..0d148f490fda 100644 --- a/packages/google-cloud-storage/tests/unit/asyncio/test_async_multi_range_downloader.py +++ b/packages/google-cloud-storage/tests/unit/asyncio/test_async_multi_range_downloader.py @@ -2631,3 +2631,99 @@ async def simulated_dl(pooled_stream, batch, *args, **kwargs): assert ps.pending_bytes == 0 await mrd.close() + + @mock.patch( + "google.cloud.storage.asyncio.async_multi_range_downloader.generate_random_56_bit_integer" + ) + @mock.patch( + "google.cloud.storage.asyncio.async_multi_range_downloader._AsyncReadObjectStream" + ) + @pytest.mark.asyncio + async def test_download_ranges_recovers_after_cancel_during_reopen( + self, mock_cls_stream, mock_random_int + ): + from google.api_core.retry_async import AsyncRetry + + from google.cloud.storage.asyncio.async_multi_range_downloader import ( + _is_read_retryable, + ) + + data = b"abcd" + mock_random_int.side_effect = [101, 201, 202] + fast_retry = AsyncRetry( + predicate=_is_read_retryable, initial=0.01, maximum=0.01 + ) + + def _stateful_stream(): + s = self._make_mock_stream() + s.is_stream_open = False + + async def _open(*args, **kwargs): + s.is_stream_open = True + + async def _close(): + if not s.is_stream_open: + raise ValueError("Stream is not open") + s.is_stream_open = False + + async def _send(req): + if not s.is_stream_open: + raise ValueError("Stream is not open") + + s.open = AsyncMock(side_effect=_open) + s.close = AsyncMock(side_effect=_close) + s.send = AsyncMock(side_effect=_send) + return s + + stream_1 = _stateful_stream() + stream_1.recv = AsyncMock(side_effect=exceptions.ServiceUnavailable("drop")) + + reopen_entered = asyncio.Event() + stream_2 = _stateful_stream() + + async def _hanging_open(*args, **kwargs): + reopen_entered.set() + await asyncio.Event().wait() + + stream_2.open = AsyncMock(side_effect=_hanging_open) + + stream_3 = _stateful_stream() + resp_q: asyncio.Queue = asyncio.Queue() + + async def _echo_send(req): + if not stream_3.is_stream_open: + raise ValueError("Stream is not open") + for rr in req.read_ranges: + await resp_q.put(_zero_copy_response(data, 0, rr.read_id, True)) + + stream_3.send = AsyncMock(side_effect=_echo_send) + stream_3.recv = AsyncMock(side_effect=resp_q.get) + + mock_cls_stream.side_effect = [stream_1, stream_2, stream_3] + + mrd = await AsyncMultiRangeDownloader.create_mrd( + self._make_mock_client(), _TEST_BUCKET_NAME, _TEST_OBJECT_NAME + ) + + buf_a = bytearray(4) + task_a = asyncio.create_task( + mrd.download_ranges([(0, 4, buf_a)], retry_policy=fast_retry) + ) + await reopen_entered.wait() + task_a.cancel() + with pytest.raises(asyncio.CancelledError): + await task_a + + buf_b, buf_c = bytearray(4), bytearray(4) + await asyncio.gather( + mrd.download_ranges([(0, 4, buf_b)], retry_policy=fast_retry), + mrd.download_ranges([(0, 4, buf_c)], retry_policy=fast_retry), + ) + assert bytes(buf_b) == data + assert bytes(buf_c) == data + assert mock_cls_stream.call_count == 3 + assert mrd.is_stream_open + + await mrd.close() + with pytest.raises(ValueError, match="Underlying bidi-gRPC stream is not open"): + await mrd.download_ranges([(0, 4, bytearray(4))], retry_policy=fast_retry) diff --git a/packages/google-cloud-storage/tests/unit/asyncio/test_stream_multiplexer.py b/packages/google-cloud-storage/tests/unit/asyncio/test_stream_multiplexer.py index 9c75b9e8e1e1..807beed07efc 100644 --- a/packages/google-cloud-storage/tests/unit/asyncio/test_stream_multiplexer.py +++ b/packages/google-cloud-storage/tests/unit/asyncio/test_stream_multiplexer.py @@ -17,6 +17,7 @@ import grpc import pytest +from google.api_core import exceptions from google.cloud import _storage_v2 from google.cloud.storage.asyncio._stream_multiplexer import ( @@ -554,6 +555,50 @@ async def test_reopen_factory_failure_leaves_generation_unchanged(self): assert mux._stream is new_stream assert mux._recv_task is not None and not mux._recv_task.done() + @pytest.mark.asyncio + async def test_send_after_cancelled_reopen_raises_retryable_error(self): + """A reopen cancelled while awaiting stream_factory leaves the closed + old stream on the multiplexer at the same generation. send() must + raise a retryable error without touching the closed stream so the next + caller's retry reopens it.""" + old_stream = AsyncMock() + old_stream.is_stream_open = True + old_stream.recv = AsyncMock(side_effect=asyncio.Event().wait) + + async def _close(): + old_stream.is_stream_open = False + + old_stream.close = AsyncMock(side_effect=_close) + mux = _StreamMultiplexer(old_stream) + mux.register({1}) + + factory_entered = asyncio.Event() + + async def hanging_factory(): + factory_entered.set() + await asyncio.Event().wait() + + reopen_task = asyncio.create_task(mux.reopen_stream(0, hanging_factory)) + await factory_entered.wait() + reopen_task.cancel() + with pytest.raises(asyncio.CancelledError): + await reopen_task + + assert mux.stream_generation == 0 + assert not old_stream.is_stream_open + + with pytest.raises(exceptions.ServiceUnavailable): + await mux.send(_storage_v2.BidiReadObjectRequest()) + assert old_stream.send.await_count == 0 + + new_stream = AsyncMock() + new_stream.is_stream_open = True + new_stream.recv = AsyncMock(side_effect=asyncio.Event().wait) + await mux.reopen_stream(0, AsyncMock(return_value=new_stream)) + + assert mux.stream_generation == 1 + assert await mux.send(_storage_v2.BidiReadObjectRequest()) == 1 + class TestClose: @pytest.mark.asyncio From ac0767afc72ff37824e9264323e76257f3f26cec Mon Sep 17 00:00:00 2001 From: zhixiangli Date: Fri, 20 Nov 2026 14:19:30 +0000 Subject: [PATCH 2/3] fix(storage): let caller cancellation propagate from AsyncMultiRangeDownloader.close --- .../asyncio/async_multi_range_downloader.py | 1 - .../test_async_multi_range_downloader.py | 22 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_multi_range_downloader.py b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_multi_range_downloader.py index 654759f4aa57..6c43b3da8fcb 100644 --- a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_multi_range_downloader.py +++ b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_multi_range_downloader.py @@ -1015,7 +1015,6 @@ async def close(self): await res except ( ValueError, - asyncio.CancelledError, exceptions.GoogleAPICallError, ): pass diff --git a/packages/google-cloud-storage/tests/unit/asyncio/test_async_multi_range_downloader.py b/packages/google-cloud-storage/tests/unit/asyncio/test_async_multi_range_downloader.py index 0d148f490fda..26fa68657ded 100644 --- a/packages/google-cloud-storage/tests/unit/asyncio/test_async_multi_range_downloader.py +++ b/packages/google-cloud-storage/tests/unit/asyncio/test_async_multi_range_downloader.py @@ -839,6 +839,28 @@ async def test_close_mrd(self, mock_cls_async_read_object_stream): assert not mrd.is_stream_open assert mrd.object_metadata is None + @mock.patch( + "google.cloud.storage.asyncio.async_multi_range_downloader._AsyncReadObjectStream" + ) + @pytest.mark.asyncio + async def test_close_propagates_caller_cancellation( + self, mock_cls_async_read_object_stream + ): + mrd, _ = await self._make_mock_mrd(mock_cls_async_read_object_stream) + close_entered = asyncio.Event() + + async def _hanging_close(): + close_entered.set() + await asyncio.Event().wait() + + mrd.read_obj_str.close = AsyncMock(side_effect=_hanging_close) + + task = asyncio.create_task(mrd.close()) + await close_entered.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + @pytest.mark.asyncio async def test_close_mrd_not_opened_should_throw_error(self): # Arrange From 96715c77b5e69c6d63f5fc4ab2f6216074d686d9 Mon Sep 17 00:00:00 2001 From: zhixiangli Date: Fri, 20 Nov 2026 14:43:24 +0000 Subject: [PATCH 3/3] fix(storage): mark MRD closed when close() is cancelled --- .../asyncio/async_multi_range_downloader.py | 42 ++++++++++--------- .../test_async_multi_range_downloader.py | 8 ++-- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_multi_range_downloader.py b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_multi_range_downloader.py index 6c43b3da8fcb..a3d10d3385e2 100644 --- a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_multi_range_downloader.py +++ b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_multi_range_downloader.py @@ -999,28 +999,30 @@ async def close(self): raise ValueError("Underlying bidi-gRPC stream is not open") async with self._trace_span("Storage.AsyncMultiRangeDownloader.close"): - if self._pool: - await self._pool.close() - self._pool = None + try: + if self._pool: + await self._pool.close() - if self._multiplexer: - await self._multiplexer.close() - self._multiplexer = None + if self._multiplexer: + await self._multiplexer.close() - if self.read_obj_str: - try: - if getattr(self.read_obj_str, "is_stream_open", True): - res = self.read_obj_str.close() - if inspect.isawaitable(res): - await res - except ( - ValueError, - exceptions.GoogleAPICallError, - ): - pass - self.read_obj_str = None - self._primary_stream = None - self._is_stream_open = False + if self.read_obj_str: + try: + if getattr(self.read_obj_str, "is_stream_open", True): + res = self.read_obj_str.close() + if inspect.isawaitable(res): + await res + except ( + ValueError, + exceptions.GoogleAPICallError, + ): + pass + finally: + self._pool = None + self._multiplexer = None + self.read_obj_str = None + self._primary_stream = None + self._is_stream_open = False @property def is_stream_open(self) -> bool: diff --git a/packages/google-cloud-storage/tests/unit/asyncio/test_async_multi_range_downloader.py b/packages/google-cloud-storage/tests/unit/asyncio/test_async_multi_range_downloader.py index 26fa68657ded..a58c797738f1 100644 --- a/packages/google-cloud-storage/tests/unit/asyncio/test_async_multi_range_downloader.py +++ b/packages/google-cloud-storage/tests/unit/asyncio/test_async_multi_range_downloader.py @@ -860,6 +860,8 @@ async def _hanging_close(): task.cancel() with pytest.raises(asyncio.CancelledError): await task + assert not mrd.is_stream_open + assert mrd.object_metadata is None @pytest.mark.asyncio async def test_close_mrd_not_opened_should_throw_error(self): @@ -2689,8 +2691,7 @@ async def _close(): s.is_stream_open = False async def _send(req): - if not s.is_stream_open: - raise ValueError("Stream is not open") + assert s.is_stream_open s.open = AsyncMock(side_effect=_open) s.close = AsyncMock(side_effect=_close) @@ -2713,8 +2714,7 @@ async def _hanging_open(*args, **kwargs): resp_q: asyncio.Queue = asyncio.Queue() async def _echo_send(req): - if not stream_3.is_stream_open: - raise ValueError("Stream is not open") + assert stream_3.is_stream_open for rr in req.read_ranges: await resp_q.put(_zero_copy_response(data, 0, rr.read_id, True))