From 5c2935f14b06276804c6b954a913013a72b831d3 Mon Sep 17 00:00:00 2001 From: zhixiangli Date: Fri, 20 Nov 2026 14:14:25 +0000 Subject: [PATCH 1/2] fix(storage): send gRPC keepalive pings on AsyncGrpcClient channels --- .../storage/asyncio/async_grpc_client.py | 24 +++++++- .../unit/asyncio/test_async_grpc_client.py | 61 ++++++++++++++++--- 2 files changed, 75 insertions(+), 10 deletions(-) diff --git a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_grpc_client.py b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_grpc_client.py index e113943857a8..ef041bde67f6 100644 --- a/packages/google-cloud-storage/google/cloud/storage/asyncio/async_grpc_client.py +++ b/packages/google-cloud-storage/google/cloud/storage/asyncio/async_grpc_client.py @@ -31,6 +31,22 @@ _rewrite_host_for_interconnect, ) +# Detect a dead connection while an RPC or bidi stream is open. Without HTTP/2 +# keepalive, a pending recv() on a bidi stream can block indefinitely when the +# path to the server silently drops traffic or the peer stops responding while +# its kernel still ACKs TCP keepalives. Ping every 30 s while an RPC is open and +# drop the connection if a ping is not ACKed within 20 s, so the stream fails +# with a retryable error instead of hanging. grpc.http2.ping_timeout_ms is set +# alongside keepalive_timeout_ms because grpc-python otherwise waits its 60 s +# default for the ping ACK. keepalive_permit_without_calls is intentionally left +# unset so idle channels send no pings (more frequent or idle pings can trigger +# GOAWAY too_many_pings from GCS frontends). +_KEEPALIVE_CHANNEL_OPTIONS: tuple[tuple[str, int], ...] = ( + ("grpc.keepalive_time_ms", 30000), + ("grpc.keepalive_timeout_ms", 20000), + ("grpc.http2.ping_timeout_ms", 20000), +) + def _validate_metadata(metadata): """Validates that metadata is a sequence of (key, value) pairs.""" @@ -192,6 +208,10 @@ def _create_async_grpc_client( host = getattr(client_options, "api_endpoint", None) or _DEFAULT_HOST quota_project_id = getattr(client_options, "quota_project_id", None) + channel_options = ( + ("grpc.primary_user_agent", primary_user_agent), + ) + _KEEPALIVE_CHANNEL_OPTIONS + if attempt_direct_path_xds_over_interconnect: host = _rewrite_host_for_interconnect(host) channel = transport_cls.create_channel( @@ -200,7 +220,7 @@ def _create_async_grpc_client( attempt_direct_path=True, attempt_direct_path_xds_over_interconnect=True, credentials=credentials, - options=(("grpc.primary_user_agent", primary_user_agent),), + options=channel_options, ) else: channel = transport_cls.create_channel( @@ -208,7 +228,7 @@ def _create_async_grpc_client( quota_project_id=quota_project_id, attempt_direct_path=attempt_direct_path, credentials=credentials, - options=(("grpc.primary_user_agent", primary_user_agent),), + options=channel_options, ) transport = transport_cls(channel=channel) diff --git a/packages/google-cloud-storage/tests/unit/asyncio/test_async_grpc_client.py b/packages/google-cloud-storage/tests/unit/asyncio/test_async_grpc_client.py index 0b96a92eb055..93b9e64e6079 100644 --- a/packages/google-cloud-storage/tests/unit/asyncio/test_async_grpc_client.py +++ b/packages/google-cloud-storage/tests/unit/asyncio/test_async_grpc_client.py @@ -24,6 +24,13 @@ from google.cloud.storage.asyncio import async_grpc_client +_EXPECTED_KEEPALIVE_OPTIONS = ( + ("grpc.keepalive_time_ms", 30000), + ("grpc.keepalive_timeout_ms", 20000), + ("grpc.http2.ping_timeout_ms", 20000), +) + + def _make_credentials(spec=None): if spec is None: return mock.Mock(spec=auth_credentials.Credentials) @@ -50,7 +57,9 @@ def test_constructor_default_options(self, mock_async_storage_client): agent_version = f"gcloud-python/{__version__}" assert agent_version in client_info.user_agent primary_user_agent = client_info.to_user_agent() - expected_options = (("grpc.primary_user_agent", primary_user_agent),) + expected_options = ( + ("grpc.primary_user_agent", primary_user_agent), + ) + _EXPECTED_KEEPALIVE_OPTIONS mock_transport_cls.create_channel.assert_called_once_with( host="storage.googleapis.com", @@ -81,7 +90,9 @@ def test_constructor_with_client_info(self, mock_async_storage_client): agent_version = f"gcloud-python/{__version__}" assert agent_version in client_info.user_agent primary_user_agent = client_info.to_user_agent() - expected_options = (("grpc.primary_user_agent", primary_user_agent),) + expected_options = ( + ("grpc.primary_user_agent", primary_user_agent), + ) + _EXPECTED_KEEPALIVE_OPTIONS mock_transport_cls.create_channel.assert_called_once_with( host="storage.googleapis.com", @@ -112,7 +123,9 @@ def test_constructor_with_quota_project_and_endpoint( kwargs = mock_async_storage_client.call_args.kwargs client_info = kwargs["client_info"] primary_user_agent = client_info.to_user_agent() - expected_options = (("grpc.primary_user_agent", primary_user_agent),) + expected_options = ( + ("grpc.primary_user_agent", primary_user_agent), + ) + _EXPECTED_KEEPALIVE_OPTIONS mock_transport_cls.create_channel.assert_called_once_with( host="custom-endpoint.com", @@ -137,7 +150,9 @@ def test_constructor_disables_directpath(self, mock_async_storage_client): agent_version = f"gcloud-python/{__version__}" assert agent_version in client_info.user_agent primary_user_agent = client_info.to_user_agent() - expected_options = (("grpc.primary_user_agent", primary_user_agent),) + expected_options = ( + ("grpc.primary_user_agent", primary_user_agent), + ) + _EXPECTED_KEEPALIVE_OPTIONS mock_transport_cls.create_channel.assert_called_once_with( host="storage.googleapis.com", @@ -282,7 +297,9 @@ def test_grpc_client_property(self, mock_grpc_gapic_client): assert "test-user-agent" in client_info.user_agent primary_user_agent = client_info.to_user_agent() - expected_options = (("grpc.primary_user_agent", primary_user_agent),) + expected_options = ( + ("grpc.primary_user_agent", primary_user_agent), + ) + _EXPECTED_KEEPALIVE_OPTIONS mock_transport_cls.create_channel.assert_called_once_with( host="storage.googleapis.com", @@ -549,7 +566,9 @@ def test_async_grpc_client_attempt_direct_path_xds_over_interconnect( kwargs = mock_async_storage_client.call_args.kwargs client_info = kwargs["client_info"] primary_user_agent = client_info.to_user_agent() - expected_options = (("grpc.primary_user_agent", primary_user_agent),) + expected_options = ( + ("grpc.primary_user_agent", primary_user_agent), + ) + _EXPECTED_KEEPALIVE_OPTIONS mock_transport_cls.create_channel.assert_called_once_with( host="storage-direct.googleapis.com", @@ -581,7 +600,9 @@ def test_create_gapic_client_custom_client_options_with_interconnect( kwargs = mock_async_storage_client.call_args.kwargs client_info = kwargs["client_info"] primary_user_agent = client_info.to_user_agent() - expected_options = (("grpc.primary_user_agent", primary_user_agent),) + expected_options = ( + ("grpc.primary_user_agent", primary_user_agent), + ) + _EXPECTED_KEEPALIVE_OPTIONS mock_transport_cls.create_channel.assert_called_once_with( host="custom.endpoint.com", @@ -613,7 +634,9 @@ def test_async_grpc_client_interconnect_env_override( kwargs = mock_async_storage_client.call_args.kwargs client_info = kwargs["client_info"] primary_user_agent = client_info.to_user_agent() - expected_options = (("grpc.primary_user_agent", primary_user_agent),) + expected_options = ( + ("grpc.primary_user_agent", primary_user_agent), + ) + _EXPECTED_KEEPALIVE_OPTIONS mock_transport_cls.create_channel.assert_called_once_with( host="storage-direct.googleapis.com", @@ -648,3 +671,25 @@ def test_async_grpc_client_interconnect_env_override( ): with pytest.raises(ValueError): async_grpc_client.AsyncGrpcClient(credentials=mock_creds) + + @pytest.mark.parametrize("attempt_direct_path_xds_over_interconnect", [False, True]) + @mock.patch("google.cloud._storage_v2.StorageAsyncClient") + def test_channel_options_send_keepalive_pings( + self, + mock_async_storage_client, + attempt_direct_path_xds_over_interconnect, + ): + mock_transport_cls = mock.MagicMock() + mock_async_storage_client.get_transport_class.return_value = mock_transport_cls + mock_creds = _make_credentials() + + async_grpc_client.AsyncGrpcClient( + credentials=mock_creds, + attempt_direct_path_xds_over_interconnect=attempt_direct_path_xds_over_interconnect, + ) + + options = dict(mock_transport_cls.create_channel.call_args.kwargs["options"]) + assert options["grpc.keepalive_time_ms"] == 30000 + assert options["grpc.keepalive_timeout_ms"] == 20000 + assert options["grpc.http2.ping_timeout_ms"] == 20000 + assert "grpc.keepalive_permit_without_calls" not in options From 9133b2c2fd07aafe62f0c00b11fc2983515b99ef Mon Sep 17 00:00:00 2001 From: zhixiangli Date: Fri, 20 Nov 2026 14:44:03 +0000 Subject: [PATCH 2/2] style(storage): fix import spacing in test_async_grpc_client --- .../tests/unit/asyncio/test_async_grpc_client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/google-cloud-storage/tests/unit/asyncio/test_async_grpc_client.py b/packages/google-cloud-storage/tests/unit/asyncio/test_async_grpc_client.py index 93b9e64e6079..b7938d04b51e 100644 --- a/packages/google-cloud-storage/tests/unit/asyncio/test_async_grpc_client.py +++ b/packages/google-cloud-storage/tests/unit/asyncio/test_async_grpc_client.py @@ -23,7 +23,6 @@ from google.cloud.storage import __version__ from google.cloud.storage.asyncio import async_grpc_client - _EXPECTED_KEEPALIVE_OPTIONS = ( ("grpc.keepalive_time_ms", 30000), ("grpc.keepalive_timeout_ms", 20000),