"""Reject unfinished Responses output before parsers or tool consumers see it.""" import asyncio import json from types import SimpleNamespace from unittest.mock import AsyncMock import httpx from langchain_core.messages import AIMessage, AIMessageChunk, HumanMessage, ToolMessage from langchain_core.runnables import RunnableLambda from pydantic import BaseModel, SecretStr import pytest import pytest_asyncio from artemis.llm.reliability import FailureCategory, LLMExhaustedError from artemis.llm.response_guard import ResponseIntegrityError, ensure_response_complete from artemis.llm.router import ModelEndpoint, ModelFactory from artemis.services import llm as llm_service from artemis.services.llm import RobustChatModelWrapper, acomplete def response_body(status, arguments, name): return { "id": "resp_test", "object": "response", "created_at": 1, "status": status, "model": "test-model", "output": [ { "type": "function_call", "id": "fc_test", "call_id": "call_test", "name": name, "arguments": arguments, "status": "completed", } ], "incomplete_details": {"reason": "max_output_tokens"} if status == "incomplete" else None, "usage": {"input_tokens": 10, "output_tokens": 3, "total_tokens": 13}, } def response_stream(status, arguments, name): body = response_body(status or "in_progress", arguments, name) events = [ {"type": "response.created", "response": {**body, "status": "in_progress", "output": []}}, { "type": "response.output_item.added", "output_index": 0, "item": {**body["output"][0], "arguments": "", "status": "in_progress"}, }, { "type": "response.function_call_arguments.delta", "output_index": 0, "item_id": "fc_test", "delta": arguments, }, ] if status: events.append({"type": "response." + status, "response": body}) return "".join("data: " + json.dumps(event) + "\n\n" for event in events) + "data: [DONE]\n\n" @pytest_asyncio.fixture async def transport_case(monkeypatch, tmp_path): from langchain_openai import ChatOpenAI import openai requests, outputs, clients, events = [], [], [], [] def handle(request): payload = json.loads(request.content) requests.append(payload) status, arguments = outputs.pop(0) name = payload.get("tools", [{"name": "finish"}])[0]["name"] if payload.get("stream"): return httpx.Response( 200, text=response_stream(status, arguments, name), headers={"content-type": "text/event-stream"}, ) return httpx.Response(200, json=response_body(status, arguments, name)) def http_client(**kwargs): client = httpx.AsyncClient(transport=httpx.MockTransport(handle), **kwargs) clients.append(client) return client monkeypatch.setattr(openai, "DefaultAsyncHttpxClient", http_client) monkeypatch.setattr(llm_service.settings, "OPENAI_API_KEY", SecretStr("test-secret")) monkeypatch.setattr(llm_service.settings, "OPENAI_BASE_URL", "https://responses.example/v1") monkeypatch.setattr(llm_service.settings, "OPENAI_MAX_RETRIES", 5) monkeypatch.setattr(llm_service.settings, "OPENAI_RETRY_INTERVAL_SECONDS", 0) monkeypatch.delenv("ARTEMIS_FAKE_LLM", raising=False) monkeypatch.setattr(llm_service, "PAUSE_FILE", tmp_path / ".artemis_paused") monkeypatch.setattr(llm_service, "_wait_for_resume", AsyncMock(return_value=False)) monkeypatch.setattr(llm_service, "_get_current_data_engine", lambda: None) monkeypatch.setattr( llm_service, "_record_llm_event", lambda *args, **kwargs: events.append(args) ) llm_service._ENDPOINT_BREAKER._states.clear() llm_service._NON_STREAMING_ENDPOINTS.clear() endpoint = ModelEndpoint(model_name="test-model", max_retries=0) model = ModelFactory.create_model(endpoint) assert isinstance(model, ChatOpenAI) yield SimpleNamespace( model=model, endpoint=endpoint, requests=requests, outputs=outputs, events=events ) for client in clients: await client.aclose() llm_service._ENDPOINT_BREAKER._states.clear() llm_service._NON_STREAMING_ENDPOINTS.clear() def wrapped(case): return RobustChatModelWrapper(case.model, endpoint=case.endpoint).bind_tools( [ { "type": "function", "function": { "name": "finish", "description": "Record the verdict", "parameters": {"type": "object", "properties": {"passed": {"type": "boolean"}}}, }, } ] ) @pytest.mark.asyncio @pytest.mark.parametrize("status", [None, "incomplete", "failed"]) @pytest.mark.parametrize("arguments", ['{"passed":true}', '{"passed":true']) async def test_unfinished_stream_never_returns_a_verdict_or_tool_call( transport_case, status, arguments ): case = transport_case case.outputs.append((status, arguments)) with pytest.raises(LLMExhaustedError) as failure: await wrapped(case).complete([HumanMessage(content="Verify the task")]) assert failure.value.failure.category is FailureCategory.INVALID_OUTPUT assert len(case.requests) == 1 resets = [payload for event, payload in case.events if event == "llm_stream_reset"] assert len(resets) == 1 and resets[0]["action"] == "discard" assert not llm_service._NON_STREAMING_ENDPOINTS @pytest.mark.asyncio @pytest.mark.parametrize("arguments", ['{"passed":true', ""]) async def test_completed_event_cannot_repair_invalid_tool_json(transport_case, arguments): case = transport_case case.outputs.append(("completed", arguments)) with pytest.raises(LLMExhaustedError) as failure: await wrapped(case).complete([]) assert failure.value.failure.category is FailureCategory.INVALID_OUTPUT assert len(case.requests) == 1 @pytest.mark.parametrize("arguments", [None, "[]", "null", '{"passed":true']) def test_original_function_call_arguments_must_be_a_complete_json_object(arguments): message = AIMessage( content=[{"type": "function_call", "arguments": arguments}], response_metadata={"model_provider": "openai", "status": "completed"}, ) with pytest.raises(ResponseIntegrityError, match="invalid_tool_arguments"): ensure_response_complete(message) @pytest.mark.asyncio async def test_completed_stream_returns_original_valid_arguments(transport_case): case = transport_case case.outputs.append(("completed", '{"passed":true}')) result = await wrapped(case).complete([]) assert result.response_metadata["status"] == "completed" assert result.tool_calls[0]["args"] == {"passed": True} @pytest.mark.asyncio @pytest.mark.parametrize("recovers", [False, True]) async def test_integrity_retry_is_bounded_and_keeps_existing_tool_evidence( transport_case, recovers ): case = transport_case case.endpoint.max_retries = 5 case.outputs.extend( [(None, '{"passed":false}'), ("completed" if recovers else None, '{"passed":true}')] ) messages = [ HumanMessage(content="Inspect then verify"), AIMessage(content="", tool_calls=[{"name": "inspect", "args": {}, "id": "prior_call"}]), ToolMessage(content="Existing screenshot evidence", tool_call_id="prior_call"), ] if recovers: result = await wrapped(case).complete(messages) assert result.tool_calls[0]["args"] == {"passed": True} else: with pytest.raises(LLMExhaustedError): await wrapped(case).complete(messages) assert len(case.requests) == 2 assert case.requests[0]["input"] == case.requests[1]["input"] assert any( item.get("call_id") == "prior_call" and item.get("output") == "Existing screenshot evidence" for item in case.requests[1]["input"] ) assert isinstance(messages[-1], ToolMessage) @pytest.mark.asyncio @pytest.mark.parametrize("status", ["incomplete", "in_progress", "queued", "cancelled"]) async def test_non_streaming_response_also_requires_completed(transport_case, status): case = transport_case case.outputs.append((status, '{"passed":true}')) with pytest.raises(LLMExhaustedError) as failure: await wrapped(case).ainvoke([]) assert failure.value.failure.category is FailureCategory.INVALID_OUTPUT @pytest.mark.asyncio @pytest.mark.parametrize("include_raw", [False, True]) async def test_structured_output_is_checked_before_returning_parsed_verdict( transport_case, include_raw ): class Verdict(BaseModel): passed: bool case = transport_case case.outputs.append(("incomplete", '{"passed":true}')) model = RobustChatModelWrapper(case.model, endpoint=case.endpoint) with pytest.raises(LLMExhaustedError): await model.with_structured_output(Verdict, include_raw=include_raw).ainvoke("Verify") @pytest.mark.asyncio async def test_downstream_parser_never_receives_incomplete_response(transport_case): case = transport_case case.outputs.append(("incomplete", '{"passed":true}')) parsed = [] chain = wrapped(case) | RunnableLambda(lambda value: parsed.append(value)) with pytest.raises(LLMExhaustedError): await chain.ainvoke([]) assert parsed == [] @pytest.mark.asyncio async def test_raw_model_compatibility_path_does_not_accept_missing_terminal_event(transport_case): case = transport_case case.outputs.append((None, '{"passed":true}')) with pytest.raises(ValueError): await acomplete(case.model, []) assert len(case.requests) == 1 @pytest.mark.asyncio async def test_cancellation_still_closes_stream_without_retry(transport_case): started, closed = asyncio.Event(), [] class StreamingModel: async def astream(self, *args, **kwargs): try: yield AIMessageChunk( content="partial", response_metadata={"model_provider": "openai"} ) started.set() await asyncio.Event().wait() finally: closed.append(True) task = asyncio.create_task(RobustChatModelWrapper(StreamingModel()).complete([])) await asyncio.wait_for(started.wait(), timeout=2) task.cancel() with pytest.raises(asyncio.CancelledError): await task assert closed == [True] assert not transport_case.requests resets = [payload for event, payload in transport_case.events if event == "llm_stream_reset"] assert len(resets) == 1 and resets[0]["reason"] == "incomplete_response"