"""Reject unfinished Responses output before parsers or tool consumers see it.""" import asyncio import json import hashlib from types import SimpleNamespace from unittest.mock import AsyncMock import httpx2 as httpx from langchain_core.messages import AIMessage, AIMessageChunk, HumanMessage, ToolMessage from langchain_core.runnables import RunnableLambda from pydantic import BaseModel, SecretStr import pytest import pytest_asyncio from artemis.llm.reliability import FailureCategory, LLMExhaustedError from artemis.llm.response_guard import ResponseIntegrityError, ensure_response_complete from artemis.llm.router import ModelEndpoint, ModelFactory from artemis.llm.diagnostics import RequestAttempt from artemis.services import llm as llm_service from artemis.services.llm import RobustChatModelWrapper, acomplete def response_body(status, arguments, name): return { "id": "resp_test", "object": "response", "created_at": 1, "status": status, "model": "test-model", "output": [ { "type": "function_call", "id": "fc_test", "call_id": "call_test", "name": name, "arguments": arguments, "status": "completed", } ], "incomplete_details": {"reason": "max_output_tokens"} if status == "incomplete" else None, "usage": {"input_tokens": 10, "output_tokens": 3, "total_tokens": 13}, } def response_stream(status, arguments, name): body = response_body(status or "in_progress", arguments, name) events = [ {"type": "response.created", "response": {**body, "status": "in_progress", "output": []}}, { "type": "response.output_item.added", "output_index": 0, "item": {**body["output"][0], "arguments": "", "status": "in_progress"}, }, { "type": "response.function_call_arguments.delta", "output_index": 0, "item_id": "fc_test", "delta": arguments, }, ] if status: events.append({"type": "response." + status, "response": body}) return ( "".join("data: " + json.dumps(event, ensure_ascii=False) + "\n\n" for event in events) + "data: [DONE]\n\n" ) class FragmentedEventStream(httpx.AsyncByteStream): """Split SSE frames and multibyte UTF-8 characters at transport boundaries.""" def __init__(self, text): self.data = text.encode("utf-8") async def __aiter__(self): for offset in range(0, len(self.data), 4093): yield self.data[offset : offset + 4093] @pytest_asyncio.fixture async def transport_case(monkeypatch, tmp_path, request): from langchain_openai import ChatOpenAI import openai requests, outputs, clients, events = [], [], [], [] def handle(http_request): payload = json.loads(http_request.content) requests.append(payload) status, arguments = outputs.pop(0) name = payload.get("tools", [{"name": "finish"}])[0]["name"] if payload.get("stream"): stream = response_stream(status, arguments, name) body = ( {"stream": FragmentedEventStream(stream)} if getattr(request, "param", None) == "fragmented" else {"text": stream} ) return httpx.Response( 200, headers={"content-type": "text/event-stream"}, **body, ) return httpx.Response(200, json=response_body(status, arguments, name)) def http_client(**kwargs): client = httpx.AsyncClient(transport=httpx.MockTransport(handle), **kwargs) clients.append(client) return client monkeypatch.setattr(openai, "DefaultAsyncHttpxClient", http_client) monkeypatch.setattr(llm_service.settings, "OPENAI_API_KEY", SecretStr("test-secret")) monkeypatch.setattr(llm_service.settings, "OPENAI_BASE_URL", "https://responses.example/v1") monkeypatch.setattr(llm_service.settings, "OPENAI_MAX_RETRIES", 5) monkeypatch.setattr(llm_service.settings, "OPENAI_RETRY_INTERVAL_SECONDS", 0) monkeypatch.delenv("ARTEMIS_FAKE_LLM", raising=False) monkeypatch.setattr(llm_service, "PAUSE_FILE", tmp_path / ".artemis_paused") monkeypatch.setattr(llm_service, "_wait_for_resume", AsyncMock(return_value=False)) monkeypatch.setattr(llm_service, "_get_current_data_engine", lambda: None) monkeypatch.setattr( llm_service, "_record_llm_event", lambda *args, **kwargs: events.append(args) ) llm_service._ENDPOINT_BREAKER._states.clear() llm_service._NON_STREAMING_ENDPOINTS.clear() endpoint = ModelEndpoint(model_name="test-model", max_retries=0) model = ModelFactory.create_model(endpoint) assert isinstance(model, ChatOpenAI) yield SimpleNamespace( model=model, endpoint=endpoint, requests=requests, outputs=outputs, events=events ) for client in clients: await client.aclose() llm_service._ENDPOINT_BREAKER._states.clear() llm_service._NON_STREAMING_ENDPOINTS.clear() def wrapped(case): return RobustChatModelWrapper(case.model, endpoint=case.endpoint).bind_tools( [ { "type": "function", "function": { "name": "finish", "description": "Record the verdict", "parameters": {"type": "object", "properties": {"passed": {"type": "boolean"}}}, }, } ] ) @pytest.mark.asyncio @pytest.mark.parametrize("status", [None, "incomplete", "failed"]) @pytest.mark.parametrize("arguments", ['{"passed":true}', '{"passed":true']) async def test_unfinished_stream_never_returns_a_verdict_or_tool_call( transport_case, status, arguments ): case = transport_case case.outputs.append((status, arguments)) with pytest.raises(LLMExhaustedError) as failure: await wrapped(case).complete([HumanMessage(content="Verify the task")]) assert failure.value.failure.category is FailureCategory.INVALID_OUTPUT assert len(case.requests) == 1 resets = [payload for event, payload in case.events if event == "llm_stream_reset"] assert len(resets) == 1 and resets[0]["action"] == "discard" assert not llm_service._NON_STREAMING_ENDPOINTS @pytest.mark.asyncio @pytest.mark.parametrize("arguments", ['{"passed":true', ""]) async def test_completed_event_cannot_repair_invalid_tool_json(transport_case, arguments): case = transport_case case.outputs.append(("completed", arguments)) with pytest.raises(LLMExhaustedError) as failure: await wrapped(case).complete([]) assert failure.value.failure.category is FailureCategory.INVALID_OUTPUT assert len(case.requests) == 1 @pytest.mark.parametrize("arguments", [None, "[]", "null", '{"passed":true']) def test_original_function_call_arguments_must_be_a_complete_json_object(arguments): message = AIMessage( content=[{"type": "function_call", "arguments": arguments}], response_metadata={"model_provider": "openai", "status": "completed"}, ) with pytest.raises(ResponseIntegrityError, match="invalid_tool_arguments"): ensure_response_complete(message) @pytest.mark.asyncio async def test_completed_stream_returns_original_valid_arguments(transport_case): case = transport_case case.outputs.append(("completed", '{"passed":true}')) result = await wrapped(case).complete([]) assert result.response_metadata["status"] == "completed" assert result.tool_calls[0]["args"] == {"passed": True} @pytest.mark.asyncio @pytest.mark.parametrize("transport_case", ["fragmented"], indirect=True) async def test_real_sdk_fingerprint_counts_received_arguments_once(transport_case): case = transport_case arguments = json.dumps({"passed": True, "evidence": "中文🙂" * 1000}, ensure_ascii=False) case.outputs.append(("completed", arguments)) observation = RequestAttempt( None, request_id="sdk-test", attempt=1, timeout=300, max_retries=0, metadata={} ) merged = None async for chunk in case.model.astream([]): observation.chunk(chunk) merged = chunk if merged is None else merged + chunk observation.finish("completed", response=merged) assert len(observation.data["tools"]) == 1 tool = observation.data["tools"][0] assert tool["call_id"] == "call_test" assert tool["argument_chars"] == len(arguments) assert tool["argument_bytes"] == len(arguments.encode()) assert tool["argument_sha256"] == hashlib.sha256(arguments.encode()).hexdigest() assert tool["argument_prefixes"][0] == { "bytes": 4096, "sha256": hashlib.sha256(arguments.encode()[:4096]).hexdigest(), } assert tool["argument_deltas"] == 1 @pytest.mark.asyncio @pytest.mark.parametrize("transport_case", ["fragmented"], indirect=True) @pytest.mark.parametrize("status", ["completed", None]) async def test_large_unicode_tool_stream_requires_completion(transport_case, status): case = transport_case arguments = {"passed": True, "evidence": "截图已核验。\n" * 65536} encoded = json.dumps(arguments, ensure_ascii=False) assert len(encoded.encode("utf-8")) > 1024 * 1024 case.outputs.append((status, encoded)) if status == "completed": result = await wrapped(case).complete([]) assert result.response_metadata["status"] == "completed" assert result.tool_calls[0]["args"] == arguments else: with pytest.raises(LLMExhaustedError) as failure: await wrapped(case).complete([]) assert failure.value.failure.category is FailureCategory.INVALID_OUTPUT assert len(case.requests) == 1 @pytest.mark.asyncio @pytest.mark.parametrize("recovers", [False, True]) @pytest.mark.parametrize("initial_status", [None, "failed"]) async def test_integrity_retry_is_bounded_and_keeps_existing_tool_evidence( transport_case, recovers, initial_status ): case = transport_case case.endpoint.max_retries = 5 case.outputs.extend( [ (initial_status, '{"passed":false}'), ("completed" if recovers else initial_status, '{"passed":true}'), ] ) messages = [ HumanMessage(content="Inspect then verify"), AIMessage(content="", tool_calls=[{"name": "inspect", "args": {}, "id": "prior_call"}]), ToolMessage(content="Existing screenshot evidence", tool_call_id="prior_call"), ] if recovers: result = await wrapped(case).complete(messages) assert result.tool_calls[0]["args"] == {"passed": True} else: with pytest.raises(LLMExhaustedError): await wrapped(case).complete(messages) assert len(case.requests) == 2 assert case.requests[0]["input"] == case.requests[1]["input"] assert any( item.get("call_id") == "prior_call" and item.get("output") == "Existing screenshot evidence" for item in case.requests[1]["input"] ) assert isinstance(messages[-1], ToolMessage) @pytest.mark.asyncio @pytest.mark.parametrize("status", ["incomplete", "in_progress", "queued", "cancelled"]) async def test_non_streaming_response_also_requires_completed(transport_case, status): case = transport_case case.outputs.append((status, '{"passed":true}')) with pytest.raises(LLMExhaustedError) as failure: await wrapped(case).ainvoke([]) assert failure.value.failure.category is FailureCategory.INVALID_OUTPUT @pytest.mark.asyncio @pytest.mark.parametrize("include_raw", [False, True]) async def test_structured_output_is_checked_before_returning_parsed_verdict( transport_case, include_raw ): class Verdict(BaseModel): passed: bool case = transport_case case.outputs.append(("incomplete", '{"passed":true}')) model = RobustChatModelWrapper(case.model, endpoint=case.endpoint) with pytest.raises(LLMExhaustedError): await model.with_structured_output(Verdict, include_raw=include_raw).ainvoke("Verify") @pytest.mark.asyncio async def test_downstream_parser_never_receives_incomplete_response(transport_case): case = transport_case case.outputs.append(("incomplete", '{"passed":true}')) parsed = [] chain = wrapped(case) | RunnableLambda(lambda value: parsed.append(value)) with pytest.raises(LLMExhaustedError): await chain.ainvoke([]) assert parsed == [] @pytest.mark.asyncio async def test_raw_model_compatibility_path_does_not_accept_missing_terminal_event(transport_case): case = transport_case case.outputs.append((None, '{"passed":true}')) with pytest.raises(ValueError): await acomplete(case.model, []) assert len(case.requests) == 1 @pytest.mark.asyncio async def test_cancellation_still_closes_stream_without_retry(transport_case): started, closed = asyncio.Event(), [] class StreamingModel: async def astream(self, *args, **kwargs): try: yield AIMessageChunk( content="partial", response_metadata={"model_provider": "openai"} ) started.set() await asyncio.Event().wait() finally: closed.append(True) task = asyncio.create_task(RobustChatModelWrapper(StreamingModel()).complete([])) await asyncio.wait_for(started.wait(), timeout=2) task.cancel() with pytest.raises(asyncio.CancelledError): await task assert closed == [True] assert not transport_case.requests resets = [payload for event, payload in transport_case.events if event == "llm_stream_reset"] assert len(resets) == 1 and resets[0]["reason"] == "incomplete_response"