Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,22 @@
_rewrite_host_for_interconnect,
)

# Detect a dead connection while an RPC or bidi stream is open. Without HTTP/2
# keepalive, a pending recv() on a bidi stream can block indefinitely when the
# path to the server silently drops traffic or the peer stops responding while
# its kernel still ACKs TCP keepalives. Ping every 30 s while an RPC is open and
# drop the connection if a ping is not ACKed within 20 s, so the stream fails
# with a retryable error instead of hanging. grpc.http2.ping_timeout_ms is set
# alongside keepalive_timeout_ms because grpc-python otherwise waits its 60 s
# default for the ping ACK. keepalive_permit_without_calls is intentionally left
# unset so idle channels send no pings (more frequent or idle pings can trigger
# GOAWAY too_many_pings from GCS frontends).
_KEEPALIVE_CHANNEL_OPTIONS: tuple[tuple[str, int], ...] = (
("grpc.keepalive_time_ms", 30000),
("grpc.keepalive_timeout_ms", 20000),
("grpc.http2.ping_timeout_ms", 20000),
)


def _validate_metadata(metadata):
"""Validates that metadata is a sequence of (key, value) pairs."""
Expand Down Expand Up @@ -192,6 +208,10 @@ def _create_async_grpc_client(
host = getattr(client_options, "api_endpoint", None) or _DEFAULT_HOST
quota_project_id = getattr(client_options, "quota_project_id", None)

channel_options = (
("grpc.primary_user_agent", primary_user_agent),
) + _KEEPALIVE_CHANNEL_OPTIONS

if attempt_direct_path_xds_over_interconnect:
host = _rewrite_host_for_interconnect(host)
channel = transport_cls.create_channel(
Expand All @@ -200,15 +220,15 @@ def _create_async_grpc_client(
attempt_direct_path=True,
attempt_direct_path_xds_over_interconnect=True,
credentials=credentials,
options=(("grpc.primary_user_agent", primary_user_agent),),
options=channel_options,
)
else:
channel = transport_cls.create_channel(
host=host,
quota_project_id=quota_project_id,
attempt_direct_path=attempt_direct_path,
credentials=credentials,
options=(("grpc.primary_user_agent", primary_user_agent),),
options=channel_options,
)
transport = transport_cls(channel=channel)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,12 @@
from google.cloud.storage import __version__
from google.cloud.storage.asyncio import async_grpc_client

_EXPECTED_KEEPALIVE_OPTIONS = (
("grpc.keepalive_time_ms", 30000),
("grpc.keepalive_timeout_ms", 20000),
("grpc.http2.ping_timeout_ms", 20000),
)


def _make_credentials(spec=None):
if spec is None:
Expand Down Expand Up @@ -50,7 +56,9 @@ def test_constructor_default_options(self, mock_async_storage_client):
agent_version = f"gcloud-python/{__version__}"
assert agent_version in client_info.user_agent
primary_user_agent = client_info.to_user_agent()
expected_options = (("grpc.primary_user_agent", primary_user_agent),)
expected_options = (
("grpc.primary_user_agent", primary_user_agent),
) + _EXPECTED_KEEPALIVE_OPTIONS

mock_transport_cls.create_channel.assert_called_once_with(
host="storage.googleapis.com",
Expand Down Expand Up @@ -81,7 +89,9 @@ def test_constructor_with_client_info(self, mock_async_storage_client):
agent_version = f"gcloud-python/{__version__}"
assert agent_version in client_info.user_agent
primary_user_agent = client_info.to_user_agent()
expected_options = (("grpc.primary_user_agent", primary_user_agent),)
expected_options = (
("grpc.primary_user_agent", primary_user_agent),
) + _EXPECTED_KEEPALIVE_OPTIONS

mock_transport_cls.create_channel.assert_called_once_with(
host="storage.googleapis.com",
Expand Down Expand Up @@ -112,7 +122,9 @@ def test_constructor_with_quota_project_and_endpoint(
kwargs = mock_async_storage_client.call_args.kwargs
client_info = kwargs["client_info"]
primary_user_agent = client_info.to_user_agent()
expected_options = (("grpc.primary_user_agent", primary_user_agent),)
expected_options = (
("grpc.primary_user_agent", primary_user_agent),
) + _EXPECTED_KEEPALIVE_OPTIONS

mock_transport_cls.create_channel.assert_called_once_with(
host="custom-endpoint.com",
Expand All @@ -137,7 +149,9 @@ def test_constructor_disables_directpath(self, mock_async_storage_client):
agent_version = f"gcloud-python/{__version__}"
assert agent_version in client_info.user_agent
primary_user_agent = client_info.to_user_agent()
expected_options = (("grpc.primary_user_agent", primary_user_agent),)
expected_options = (
("grpc.primary_user_agent", primary_user_agent),
) + _EXPECTED_KEEPALIVE_OPTIONS

mock_transport_cls.create_channel.assert_called_once_with(
host="storage.googleapis.com",
Expand Down Expand Up @@ -282,7 +296,9 @@ def test_grpc_client_property(self, mock_grpc_gapic_client):
assert "test-user-agent" in client_info.user_agent

primary_user_agent = client_info.to_user_agent()
expected_options = (("grpc.primary_user_agent", primary_user_agent),)
expected_options = (
("grpc.primary_user_agent", primary_user_agent),
) + _EXPECTED_KEEPALIVE_OPTIONS

mock_transport_cls.create_channel.assert_called_once_with(
host="storage.googleapis.com",
Expand Down Expand Up @@ -549,7 +565,9 @@ def test_async_grpc_client_attempt_direct_path_xds_over_interconnect(
kwargs = mock_async_storage_client.call_args.kwargs
client_info = kwargs["client_info"]
primary_user_agent = client_info.to_user_agent()
expected_options = (("grpc.primary_user_agent", primary_user_agent),)
expected_options = (
("grpc.primary_user_agent", primary_user_agent),
) + _EXPECTED_KEEPALIVE_OPTIONS

mock_transport_cls.create_channel.assert_called_once_with(
host="storage-direct.googleapis.com",
Expand Down Expand Up @@ -581,7 +599,9 @@ def test_create_gapic_client_custom_client_options_with_interconnect(
kwargs = mock_async_storage_client.call_args.kwargs
client_info = kwargs["client_info"]
primary_user_agent = client_info.to_user_agent()
expected_options = (("grpc.primary_user_agent", primary_user_agent),)
expected_options = (
("grpc.primary_user_agent", primary_user_agent),
) + _EXPECTED_KEEPALIVE_OPTIONS

mock_transport_cls.create_channel.assert_called_once_with(
host="custom.endpoint.com",
Expand Down Expand Up @@ -613,7 +633,9 @@ def test_async_grpc_client_interconnect_env_override(
kwargs = mock_async_storage_client.call_args.kwargs
client_info = kwargs["client_info"]
primary_user_agent = client_info.to_user_agent()
expected_options = (("grpc.primary_user_agent", primary_user_agent),)
expected_options = (
("grpc.primary_user_agent", primary_user_agent),
) + _EXPECTED_KEEPALIVE_OPTIONS

mock_transport_cls.create_channel.assert_called_once_with(
host="storage-direct.googleapis.com",
Expand Down Expand Up @@ -648,3 +670,25 @@ def test_async_grpc_client_interconnect_env_override(
):
with pytest.raises(ValueError):
async_grpc_client.AsyncGrpcClient(credentials=mock_creds)

@pytest.mark.parametrize("attempt_direct_path_xds_over_interconnect", [False, True])
@mock.patch("google.cloud._storage_v2.StorageAsyncClient")
def test_channel_options_send_keepalive_pings(
self,
mock_async_storage_client,
attempt_direct_path_xds_over_interconnect,
):
mock_transport_cls = mock.MagicMock()
mock_async_storage_client.get_transport_class.return_value = mock_transport_cls
mock_creds = _make_credentials()

async_grpc_client.AsyncGrpcClient(
credentials=mock_creds,
attempt_direct_path_xds_over_interconnect=attempt_direct_path_xds_over_interconnect,
)

options = dict(mock_transport_cls.create_channel.call_args.kwargs["options"])
assert options["grpc.keepalive_time_ms"] == 30000
assert options["grpc.keepalive_timeout_ms"] == 20000
assert options["grpc.http2.ping_timeout_ms"] == 20000
assert "grpc.keepalive_permit_without_calls" not in options
Loading