From a0ab27049830051837b93eb1d87cd72833fad8e8 Mon Sep 17 00:00:00 2001 From: Soroush Baraouf Date: Tue, 17 Nov 2026 22:00:03 -0500 Subject: [PATCH] fix(client): decode SSE streams as UTF-8 Read SSE response bytes and decode complete lines as UTF-8. Preserve CR, LF and CRLF framing across chunks, regardless of HTTP charset hints or client defaults. --- src/a2a/client/transports/http_helpers.py | 22 +++--- tests/client/transports/test_http_helpers.py | 70 +++++++++++++++---- .../client/transports/test_jsonrpc_client.py | 18 ++--- tests/client/transports/test_rest_client.py | 20 +++--- 4 files changed, 89 insertions(+), 41 deletions(-) diff --git a/src/a2a/client/transports/http_helpers.py b/src/a2a/client/transports/http_helpers.py index 17e882b24..d1ee7caee 100644 --- a/src/a2a/client/transports/http_helpers.py +++ b/src/a2a/client/transports/http_helpers.py @@ -71,7 +71,7 @@ async def send_http_request( return response.json() -_SSE_LINE_END = re.compile(r'\r\n|\r|\n') +_SSE_LINE_END = re.compile(rb'\r\n|\r|\n') async def _aiter_sse_lines( @@ -83,25 +83,27 @@ async def _aiter_sse_lines( only the new chunk is scanned, so a long `data:` line that spans many chunks costs linear rather than quadratic time. """ - pieces: list[str] = [] + # SSE is always UTF-8, regardless of HTTP charset and client defaults. + # CR and LF cannot split a UTF-8 code point, so decode complete lines. + pieces: list[bytes] = [] held_cr = False - async for chunk in response.aiter_text(): - text = '\r' + chunk if held_cr else chunk + async for chunk in response.aiter_bytes(): + data = b'\r' + chunk if held_cr else chunk # Hold a trailing CR back: the next chunk may start with its LF. - held_cr = text.endswith('\r') - *lines, rest = _SSE_LINE_END.split(text[:-1] if held_cr else text) + held_cr = data.endswith(b'\r') + *lines, rest = _SSE_LINE_END.split(data[:-1] if held_cr else data) if lines: - lines[0] = ''.join(pieces) + lines[0] + lines[0] = b''.join(pieces) + lines[0] pieces = [] for line in lines: - yield line + yield line.decode('utf-8', errors='replace') if rest: pieces.append(rest) - tail = ''.join(pieces) + tail = b''.join(pieces) # A held CR at end of stream still ends a line, even an empty one. That # blank line is what dispatches the last event, so yield it too. if held_cr or tail: - yield tail + yield tail.decode('utf-8', errors='replace') async def parse_sse_stream( diff --git a/tests/client/transports/test_http_helpers.py b/tests/client/transports/test_http_helpers.py index 84a0ed2e2..e48a3876f 100644 --- a/tests/client/transports/test_http_helpers.py +++ b/tests/client/transports/test_http_helpers.py @@ -26,18 +26,21 @@ def test_default_sse_error_handler(): @pytest.mark.asyncio async def test_parse_sse_stream_edge_cases(): - async def mock_aiter_text(): - yield ': comment line (should be ignored)\n' - yield 'event: custom_event\n' - yield 'data: hello\n' - yield 'data: world\n' - yield '\n' - yield '\n' - yield 'data: \n' - yield '\n' - - response = httpx_lib.Response(200) - response.aiter_text = mock_aiter_text # type: ignore + response = httpx_lib.Response( + 200, + content=_byte_chunks( + [ + b': comment line (should be ignored)\n', + b'event: custom_event\n', + b'data: hello\n', + b'data: world\n', + b'\n', + b'\n', + b'data: \n', + b'\n', + ] + ), + ) events = [e async for e in parse_sse_stream(response)] assert events == [ @@ -51,6 +54,49 @@ async def _byte_chunks(chunks: list[bytes]) -> AsyncIterator[bytes]: yield chunk +@pytest.mark.asyncio +@pytest.mark.parametrize( + ('content_type', 'default_encoding'), + [ + pytest.param('text/event-stream', 'utf-8', id='utf8-control'), + pytest.param( + 'text/event-stream; charset=iso-8859-1', + 'utf-8', + id='response-charset', + ), + pytest.param( + 'text/event-stream', 'iso-8859-1', id='client-default-encoding' + ), + ], +) +async def test_send_http_stream_request_sse_uses_utf8( + content_type: str, default_encoding: str +) -> None: + """Decode SSE as UTF-8 even when HTTP text decoding selects another charset.""" + body = 'data: café 東京\r\n\r\n'.encode('utf-8') + chunks = [body[i : i + 1] for i in range(len(body))] + + def respond(request: httpx_lib.Request) -> httpx_lib.Response: + return httpx_lib.Response( + 200, + headers={'Content-Type': content_type}, + content=_byte_chunks(chunks), + ) + + async with httpx_lib.AsyncClient( + transport=httpx_lib.MockTransport(respond), + default_encoding=default_encoding, + ) as client: + events = [ + event + async for event in send_http_stream_request( + client, 'GET', 'https://agent.example' + ) + ] + + assert events == ['café 東京'] + + @pytest.mark.asyncio @pytest.mark.parametrize( ('chunks', 'expected'), diff --git a/tests/client/transports/test_jsonrpc_client.py b/tests/client/transports/test_jsonrpc_client.py index 3ae287d23..4566bb3e3 100644 --- a/tests/client/transports/test_jsonrpc_client.py +++ b/tests/client/transports/test_jsonrpc_client.py @@ -454,8 +454,8 @@ async def test_send_message_streaming_sse_error( mock_response = AsyncMock(spec=httpx_lib.Response) mock_response.raise_for_status = MagicMock() mock_response.headers = {'content-type': 'text/event-stream'} - mock_response.aiter_text.return_value = async_iterable_from_list( - ['event: error\ndata: Simulated SSE error\n\n'] + mock_response.aiter_bytes.return_value = async_iterable_from_list( + [b'event: error\ndata: Simulated SSE error\n\n'] ) mock_aconnect_sse.return_value.__aenter__.return_value = mock_response @@ -474,7 +474,7 @@ async def test_send_message_streaming_request_error( mock_response = AsyncMock(spec=httpx_lib.Response) mock_response.raise_for_status = MagicMock() mock_response.headers = {'content-type': 'text/event-stream'} - mock_response.aiter_text.side_effect = httpx_lib.RequestError( + mock_response.aiter_bytes.side_effect = httpx_lib.RequestError( 'Simulated request error', request=MagicMock() ) mock_aconnect_sse.return_value.__aenter__.return_value = mock_response @@ -494,7 +494,7 @@ async def test_send_message_streaming_timeout( mock_response = AsyncMock(spec=httpx_lib.Response) mock_response.raise_for_status = MagicMock() mock_response.headers = {'content-type': 'text/event-stream'} - mock_response.aiter_text.side_effect = httpx_lib.TimeoutException( + mock_response.aiter_bytes.side_effect = httpx_lib.TimeoutException( 'Timeout' ) mock_aconnect_sse.return_value.__aenter__.return_value = mock_response @@ -580,7 +580,7 @@ async def test_send_message_streaming_server_error_propagates( request=httpx_lib.Request('POST', 'http://test.url'), response=mock_response, ) - mock_response.aiter_text.return_value = async_iterable_from_list([]) + mock_response.aiter_bytes.return_value = async_iterable_from_list([]) mock_aconnect_sse.return_value.__aenter__.return_value = mock_response with pytest.raises(A2AClientError) as exc_info: @@ -812,8 +812,8 @@ async def test_send_message_streaming_ignores_unrecognized_fields( mock_response = AsyncMock(spec=httpx_lib.Response) mock_response.raise_for_status = MagicMock() mock_response.headers = {'content-type': 'text/event-stream'} - mock_response.aiter_text.return_value = async_iterable_from_list( - [f'data: {json.dumps(payload)}\n\n'] + mock_response.aiter_bytes.return_value = async_iterable_from_list( + [f'data: {json.dumps(payload)}\n\n'.encode()] ) mock_sse_source.return_value.__aenter__.return_value = mock_response @@ -968,8 +968,8 @@ async def test_streaming_malformed_event_raises_a2a_error( mock_response = AsyncMock(spec=httpx_lib.Response) mock_response.raise_for_status = MagicMock() mock_response.headers = {'content-type': 'text/event-stream'} - mock_response.aiter_text.return_value = async_iterable_from_list( - [f'{line}\n' for line in lines] + mock_response.aiter_bytes.return_value = async_iterable_from_list( + [f'{line}\n'.encode() for line in lines] ) mock_sse_source.return_value.__aenter__.return_value = mock_response diff --git a/tests/client/transports/test_rest_client.py b/tests/client/transports/test_rest_client.py index 0163da021..ebc3f93d5 100644 --- a/tests/client/transports/test_rest_client.py +++ b/tests/client/transports/test_rest_client.py @@ -112,7 +112,7 @@ async def test_send_message_streaming_timeout( mock_response = AsyncMock(spec=httpx_lib.Response) mock_response.headers = {'content-type': 'text/event-stream'} mock_response.raise_for_status.return_value = None - mock_response.aiter_text.side_effect = httpx_lib.TimeoutException( + mock_response.aiter_bytes.side_effect = httpx_lib.TimeoutException( 'Read timed out' ) mock_aconnect_sse.return_value.__aenter__.return_value = mock_response @@ -351,7 +351,7 @@ async def test_send_message_streaming_with_new_extensions( mock_response = AsyncMock(spec=httpx_lib.Response) mock_response.headers = {'content-type': 'text/event-stream'} mock_response.raise_for_status.return_value = None - mock_response.aiter_text.return_value = async_iterable_from_list([]) + mock_response.aiter_bytes.return_value = async_iterable_from_list([]) mock_aconnect_sse.return_value.__aenter__.return_value = mock_response context = ClientCallContext( @@ -401,7 +401,7 @@ async def test_send_message_streaming_server_error_propagates( request=httpx_lib.Request('POST', 'http://test.url'), response=mock_response, ) - mock_response.aiter_text.return_value = async_iterable_from_list([]) + mock_response.aiter_bytes.return_value = async_iterable_from_list([]) mock_aconnect_sse.return_value.__aenter__.return_value = mock_response with pytest.raises(A2AClientError) as exc_info: @@ -836,7 +836,7 @@ async def test_rest_streaming_methods_prepend_tenant( # noqa: PLR0913 mock_response = AsyncMock(spec=httpx_lib.Response) mock_response.headers = {'content-type': 'text/event-stream'} mock_response.raise_for_status.return_value = None - mock_response.aiter_text.return_value = async_iterable_from_list([]) + mock_response.aiter_bytes.return_value = async_iterable_from_list([]) mock_aconnect_sse.return_value.__aenter__.return_value = mock_response # 3. Call the method @@ -989,8 +989,8 @@ async def test_send_message_streaming_ignores_unrecognized_fields( mock_response = AsyncMock(spec=httpx_lib.Response) mock_response.raise_for_status = MagicMock() mock_response.headers = {'content-type': 'text/event-stream'} - mock_response.aiter_text.return_value = async_iterable_from_list( - [f'data: {json.dumps(payload)}\n\n'] + mock_response.aiter_bytes.return_value = async_iterable_from_list( + [f'data: {json.dumps(payload)}\n\n'.encode()] ) mock_sse_source.return_value.__aenter__.return_value = mock_response @@ -1074,8 +1074,8 @@ async def test_stream_error_raises_mapped_error( mock_response = AsyncMock(spec=httpx_lib.Response) mock_response.raise_for_status = MagicMock() mock_response.headers = {'content-type': 'text/event-stream'} - mock_response.aiter_text.return_value = async_iterable_from_list( - [sse_text] + mock_response.aiter_bytes.return_value = async_iterable_from_list( + [sse_text.encode()] ) mock_sse_source.return_value.__aenter__.return_value = mock_response @@ -1100,8 +1100,8 @@ async def test_stream_event_of_unknown_type_is_still_yielded( mock_response = AsyncMock(spec=httpx_lib.Response) mock_response.raise_for_status = MagicMock() mock_response.headers = {'content-type': 'text/event-stream'} - mock_response.aiter_text.return_value = async_iterable_from_list( - [f'data: {json.dumps({"someNewEvent": {"id": "e1"}})}\n\n'] + mock_response.aiter_bytes.return_value = async_iterable_from_list( + [f'data: {json.dumps({"someNewEvent": {"id": "e1"}})}\n\n'.encode()] ) mock_sse_source.return_value.__aenter__.return_value = mock_response