Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
from typing import Awaitable, Callable, Dict, Optional, Set

import grpc
from google.api_core import exceptions

from google.cloud import _storage_v2
from google.cloud.storage.asyncio.async_read_object_stream import (
Expand Down Expand Up @@ -181,6 +182,13 @@ async def _recv_loop(self) -> None:
self._put_error_nowait(queue, error)

async def send(self, request: _storage_v2.BidiReadObjectRequest) -> int:
if not self._stream.is_stream_open:
# A reopen cancelled after closing the old stream leaves that
# closed stream on the multiplexer at the same generation; raise a
# retryable error so the caller's retry reopens it.
raise exceptions.ServiceUnavailable(
"Bidi read stream is not open; it will be reopened"
)
self._ensure_recv_loop()
await self._stream.send(request)
return self._stream_generation
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -999,29 +999,30 @@ async def close(self):
raise ValueError("Underlying bidi-gRPC stream is not open")

async with self._trace_span("Storage.AsyncMultiRangeDownloader.close"):
if self._pool:
await self._pool.close()
self._pool = None
try:
if self._pool:
await self._pool.close()

if self._multiplexer:
await self._multiplexer.close()
self._multiplexer = None
if self._multiplexer:
await self._multiplexer.close()

if self.read_obj_str:
try:
if getattr(self.read_obj_str, "is_stream_open", True):
res = self.read_obj_str.close()
if inspect.isawaitable(res):
await res
except (
ValueError,
asyncio.CancelledError,
exceptions.GoogleAPICallError,
):
pass
self.read_obj_str = None
self._primary_stream = None
self._is_stream_open = False
if self.read_obj_str:
try:
if getattr(self.read_obj_str, "is_stream_open", True):
res = self.read_obj_str.close()
if inspect.isawaitable(res):
await res
except (
ValueError,
exceptions.GoogleAPICallError,
):
pass
finally:
self._pool = None
self._multiplexer = None
self.read_obj_str = None
self._primary_stream = None
self._is_stream_open = False
Comment on lines +1002 to +1025

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

high

When closing multiple resources during cleanup, wrap each close call in a try-except block to handle exceptions individually, log any failures as warnings, and use a finally block to clear the resource references. This ensures that a failure in closing one resource does not prevent other resources from being closed.

            try:
                if self._pool:
                    await self._pool.close()
            except Exception as e:
                logger.warning("Failed to close pool: %s", e)

            try:
                if self._multiplexer:
                    await self._multiplexer.close()
            except Exception as e:
                logger.warning("Failed to close multiplexer: %s", e)

            try:
                if self.read_obj_str:
                    if getattr(self.read_obj_str, 'is_stream_open', True):
                        res = self.read_obj_str.close()
                        if inspect.isawaitable(res):
                            await res
            except Exception as e:
                logger.warning("Failed to close stream: %s", e)
            finally:
                self._pool = None
                self._multiplexer = None
                self.read_obj_str = None
                self._primary_stream = None
                self._is_stream_open = False
References
  1. When closing multiple resources during cleanup, wrap each close call in a try-except block to handle exceptions individually, log any failures as warnings, and use a finally block to clear the resource references.


@property
def is_stream_open(self) -> bool:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -839,6 +839,30 @@ async def test_close_mrd(self, mock_cls_async_read_object_stream):
assert not mrd.is_stream_open
assert mrd.object_metadata is None

@mock.patch(
"google.cloud.storage.asyncio.async_multi_range_downloader._AsyncReadObjectStream"
)
@pytest.mark.asyncio
async def test_close_propagates_caller_cancellation(
self, mock_cls_async_read_object_stream
):
mrd, _ = await self._make_mock_mrd(mock_cls_async_read_object_stream)
close_entered = asyncio.Event()

async def _hanging_close():
close_entered.set()
await asyncio.Event().wait()

mrd.read_obj_str.close = AsyncMock(side_effect=_hanging_close)

task = asyncio.create_task(mrd.close())
await close_entered.wait()
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
assert not mrd.is_stream_open
assert mrd.object_metadata is None

@pytest.mark.asyncio
async def test_close_mrd_not_opened_should_throw_error(self):
# Arrange
Expand Down Expand Up @@ -2631,3 +2655,97 @@ async def simulated_dl(pooled_stream, batch, *args, **kwargs):
assert ps.pending_bytes == 0

await mrd.close()

@mock.patch(
"google.cloud.storage.asyncio.async_multi_range_downloader.generate_random_56_bit_integer"
)
@mock.patch(
"google.cloud.storage.asyncio.async_multi_range_downloader._AsyncReadObjectStream"
)
@pytest.mark.asyncio
async def test_download_ranges_recovers_after_cancel_during_reopen(
self, mock_cls_stream, mock_random_int
):
from google.api_core.retry_async import AsyncRetry

from google.cloud.storage.asyncio.async_multi_range_downloader import (
_is_read_retryable,
)

data = b"abcd"
mock_random_int.side_effect = [101, 201, 202]
fast_retry = AsyncRetry(
predicate=_is_read_retryable, initial=0.01, maximum=0.01
)

def _stateful_stream():
s = self._make_mock_stream()
s.is_stream_open = False

async def _open(*args, **kwargs):
s.is_stream_open = True

async def _close():
if not s.is_stream_open:
raise ValueError("Stream is not open")
s.is_stream_open = False

async def _send(req):
assert s.is_stream_open

s.open = AsyncMock(side_effect=_open)
s.close = AsyncMock(side_effect=_close)
s.send = AsyncMock(side_effect=_send)
return s

stream_1 = _stateful_stream()
stream_1.recv = AsyncMock(side_effect=exceptions.ServiceUnavailable("drop"))

reopen_entered = asyncio.Event()
stream_2 = _stateful_stream()

async def _hanging_open(*args, **kwargs):
reopen_entered.set()
await asyncio.Event().wait()

stream_2.open = AsyncMock(side_effect=_hanging_open)

stream_3 = _stateful_stream()
resp_q: asyncio.Queue = asyncio.Queue()

async def _echo_send(req):
assert stream_3.is_stream_open
for rr in req.read_ranges:
await resp_q.put(_zero_copy_response(data, 0, rr.read_id, True))

stream_3.send = AsyncMock(side_effect=_echo_send)
stream_3.recv = AsyncMock(side_effect=resp_q.get)

mock_cls_stream.side_effect = [stream_1, stream_2, stream_3]

mrd = await AsyncMultiRangeDownloader.create_mrd(
self._make_mock_client(), _TEST_BUCKET_NAME, _TEST_OBJECT_NAME
)

buf_a = bytearray(4)
task_a = asyncio.create_task(
mrd.download_ranges([(0, 4, buf_a)], retry_policy=fast_retry)
)
await reopen_entered.wait()
task_a.cancel()
with pytest.raises(asyncio.CancelledError):
await task_a

buf_b, buf_c = bytearray(4), bytearray(4)
await asyncio.gather(
mrd.download_ranges([(0, 4, buf_b)], retry_policy=fast_retry),
mrd.download_ranges([(0, 4, buf_c)], retry_policy=fast_retry),
)
assert bytes(buf_b) == data
assert bytes(buf_c) == data
assert mock_cls_stream.call_count == 3
assert mrd.is_stream_open

await mrd.close()
with pytest.raises(ValueError, match="Underlying bidi-gRPC stream is not open"):
await mrd.download_ranges([(0, 4, bytearray(4))], retry_policy=fast_retry)
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

import grpc
import pytest
from google.api_core import exceptions

from google.cloud import _storage_v2
from google.cloud.storage.asyncio._stream_multiplexer import (
Expand Down Expand Up @@ -554,6 +555,50 @@ async def test_reopen_factory_failure_leaves_generation_unchanged(self):
assert mux._stream is new_stream
assert mux._recv_task is not None and not mux._recv_task.done()

@pytest.mark.asyncio
async def test_send_after_cancelled_reopen_raises_retryable_error(self):
"""A reopen cancelled while awaiting stream_factory leaves the closed
old stream on the multiplexer at the same generation. send() must
raise a retryable error without touching the closed stream so the next
caller's retry reopens it."""
old_stream = AsyncMock()
old_stream.is_stream_open = True
old_stream.recv = AsyncMock(side_effect=asyncio.Event().wait)

async def _close():
old_stream.is_stream_open = False

old_stream.close = AsyncMock(side_effect=_close)
mux = _StreamMultiplexer(old_stream)
mux.register({1})

factory_entered = asyncio.Event()

async def hanging_factory():
factory_entered.set()
await asyncio.Event().wait()

reopen_task = asyncio.create_task(mux.reopen_stream(0, hanging_factory))
await factory_entered.wait()
reopen_task.cancel()
with pytest.raises(asyncio.CancelledError):
await reopen_task

assert mux.stream_generation == 0
assert not old_stream.is_stream_open

with pytest.raises(exceptions.ServiceUnavailable):
await mux.send(_storage_v2.BidiReadObjectRequest())
assert old_stream.send.await_count == 0

new_stream = AsyncMock()
new_stream.is_stream_open = True
new_stream.recv = AsyncMock(side_effect=asyncio.Event().wait)
await mux.reopen_stream(0, AsyncMock(return_value=new_stream))

assert mux.stream_generation == 1
assert await mux.send(_storage_v2.BidiReadObjectRequest()) == 1


class TestClose:
@pytest.mark.asyncio
Expand Down
Loading