"""Transport-level checks: every model call must use /responses.""" import json from unittest.mock import MagicMock import httpx import pytest from langchain_core.messages import HumanMessage, SystemMessage, ToolMessage from pydantic import BaseModel, SecretStr, ValidationError from artemis.config.settings import settings from artemis.llm.messages import adapt_responses_messages from artemis.llm.router import ModelEndpoint, ModelFactory, ModelProvider from artemis.services.llm import RobustChatModelWrapper def response_body(output): return { "id": "resp_test", "object": "response", "created_at": 1, "status": "completed", "model": "test-model", "output": output, "usage": {"input_tokens": 10, "output_tokens": 3, "total_tokens": 13}, } def text_output(text): return [ { "id": "msg_test", "type": "message", "role": "assistant", "status": "completed", "content": [{"type": "output_text", "text": text, "annotations": []}], } ] @pytest.fixture def transport_model(monkeypatch): from langchain_openai import ChatOpenAI import openai requests = [] outputs = [] def handle(request): requests.append(request) return httpx.Response( 200, json=response_body(outputs.pop(0) if outputs else text_output("OK")) ) monkeypatch.setattr( openai, "DefaultAsyncHttpxClient", lambda **kwargs: httpx.AsyncClient(transport=httpx.MockTransport(handle), **kwargs), ) monkeypatch.setattr(settings, "OPENAI_API_KEY", SecretStr("test-secret")) monkeypatch.setattr(settings, "OPENAI_BASE_URL", "https://responses.example/v1") monkeypatch.delenv("ARTEMIS_FAKE_LLM", raising=False) model = ModelFactory.create_model(ModelEndpoint(model_name="test-model", timeout_seconds=23)) assert isinstance(model, ChatOpenAI) return model, requests, outputs @pytest.mark.asyncio async def test_text_and_image_use_responses_transport(transport_model): model, requests, _ = transport_model result = await model.ainvoke( [ HumanMessage( content=[ {"type": "text", "text": "Describe"}, {"type": "image_url", "image_url": {"url": "data:image/png;base64,AA=="}}, ] ) ] ) assert requests[0].url == "https://responses.example/v1/responses" payload = json.loads(requests[0].content) assert payload["store"] is False assert payload["input"][0]["content"][1]["type"] == "input_image" assert "messages" not in payload assert result.usage_metadata["total_tokens"] == 13 @pytest.mark.asyncio @pytest.mark.parametrize( ("model_name", "reasoning_effort"), [ ("mimo-v2.6-pro", "high"), ("mimo-v2.6-flash", "high"), ("gpt-6-luna", "max"), ("gpt-6-luna", "high"), ], ) async def test_selected_models_put_instructions_in_user_input( transport_model, model_name, reasoning_effort ): _, requests, _ = transport_model endpoint = ModelEndpoint( model_name=model_name, reasoning_effort=reasoning_effort, timeout_seconds=23, max_retries=0, ) model = ModelFactory.create_model(endpoint) wrapper = RobustChatModelWrapper(model, endpoint=endpoint) tool = { "type": "function", "function": { "name": "inspect_screen", "description": "Inspect the current screen", "parameters": {"type": "object", "properties": {}}, }, } await wrapper.bind_tools([tool]).ainvoke( [ SystemMessage(content="Use the available tools."), HumanMessage( content=[ {"type": "text", "text": "Inspect this screen."}, {"type": "image_url", "image_url": {"url": "data:image/png;base64,AA=="}}, ] ), ] ) payload = json.loads(requests[-1].content) assert payload["model"] == model_name assert payload["reasoning"]["effort"] == reasoning_effort assert payload["input"][0]["role"] == "user" assert payload["input"][0]["content"][0]["text"] == "Use the available tools." assert payload["input"][0]["content"][2]["type"] == "input_image" assert payload["tools"][0]["name"] == "inspect_screen" assert all(item.get("role") != "system" for item in payload["input"]) @pytest.mark.parametrize("model_name", ["gpt-6-luna", "GPT-5.6-luna"]) def test_gpt_message_adaptation_keeps_history_and_images(model_name): image = {"type": "image_url", "image_url": {"url": "data:image/png;base64,AA=="}} original = [ SystemMessage(content="First instruction"), SystemMessage(content="Second instruction"), HumanMessage(content=[{"type": "text", "text": "Observation"}, image]), SystemMessage(content="Follow-up instruction"), ] adapted = adapt_responses_messages(original, model_name) assert len(adapted) == 2 assert isinstance(adapted[0], HumanMessage) assert adapted[0].content[0]["text"] == "First instruction" assert adapted[0].content[1]["text"] == "Second instruction" assert adapted[0].content[3] == image assert isinstance(adapted[1], HumanMessage) assert adapted[1].content == [{"type": "text", "text": "Follow-up instruction"}] assert isinstance(original[0], SystemMessage) assert len(original[2].content) == 2 @pytest.mark.parametrize("model_name", ["mimo-v2.6-pro", "mimo-v2.6-flash"]) def test_mimo_messages_use_same_user_role_format(model_name): messages = [SystemMessage(content="Instructions"), HumanMessage(content="Task")] adapted = adapt_responses_messages(messages, model_name) assert len(adapted) == 1 assert isinstance(adapted[0], HumanMessage) assert adapted[0].content[0]["text"] == "Instructions" assert adapted[0].content[1]["text"] == "Task" assert isinstance(messages[0], SystemMessage) @pytest.mark.asyncio async def test_max_reasoning_is_sent_unchanged_to_responses(transport_model): model, requests, _ = transport_model model.reasoning_effort = "max" await model.ainvoke([HumanMessage(content="Check")]) assert requests[0].url.path == "/v1/responses" assert json.loads(requests[0].content)["reasoning"]["effort"] == "max" @pytest.mark.asyncio async def test_function_call_roundtrip_uses_call_id(transport_model): model, requests, outputs = transport_model outputs.append( [ { "type": "function_call", "name": "lookup", "call_id": "call_1", "id": "fc_1", "arguments": '{"query":"hello"}', "status": "completed", } ] ) tool = { "type": "function", "function": { "name": "lookup", "description": "Look up a value", "parameters": { "type": "object", "properties": {"query": {"type": "string"}}, "required": ["query"], }, }, } messages = [HumanMessage(content="Look up hello")] answer = await model.bind_tools([tool]).ainvoke(messages) assert answer.tool_calls[0]["id"] == "call_1" await model.ainvoke([*messages, answer, ToolMessage(content="found", tool_call_id="call_1")]) payload = json.loads(requests[1].content) assert any( item.get("type") == "function_call_output" and item["call_id"] == "call_1" for item in payload["input"] ) assert all(r.url.path == "/v1/responses" for r in requests) @pytest.mark.asyncio async def test_structured_output_uses_responses_text_format(transport_model): class Verdict(BaseModel): ok: bool model, requests, outputs = transport_model outputs.append(text_output('{"ok":true}')) result = await model.with_structured_output(Verdict, method="json_schema", strict=True).ainvoke( "Check" ) assert result.ok payload = json.loads(requests[0].content) assert payload["text"]["format"]["type"] == "json_schema" assert "response_format" not in payload @pytest.mark.parametrize( "provider", ["google", "gemini", "anthropic", "openrouter", "ollama", "custom"] ) def test_other_provider_transports_are_rejected(provider): with pytest.raises(ValueError, match="only OpenAI Responses"): ModelProvider.from_string(provider) with pytest.raises(ValidationError): ModelEndpoint(provider=provider) @pytest.mark.parametrize("max_retries", [0, 3]) def test_client_settings_and_cache_credentials(monkeypatch, max_retries): monkeypatch.setattr(settings, "OPENAI_API_KEY", SecretStr("first")) monkeypatch.setattr(settings, "OPENAI_MAX_RETRIES", max_retries) endpoint = ModelEndpoint(api_key="test", model_name="gpt-5-mini", reasoning_effort="low") model = ModelFactory.create_model(endpoint) assert model.use_responses_api is True assert model.max_retries == 0 # Gateway owns retries, never multiply at SDK layer. assert model.reasoning_effort == "low" dynamic = ModelEndpoint() key = dynamic.cache_key() monkeypatch.setattr(settings, "OPENAI_API_KEY", SecretStr("second")) assert dynamic.cache_key() != key assert "first" not in repr(key) def test_web_search_uses_openai_tool(): base = MagicMock() wrapper = RobustChatModelWrapper(base, endpoint=ModelEndpoint(enable_grounding=True)) wrapper.bind_tools([]) base.bind_tools.assert_called_once_with([{"type": "web_search"}]) def test_video_text_fallback_extracts_responses_blocks(): from langchain_core.messages import AIMessage from artemis.agents.video_analyzer.universal_engine import _extract_submit_payload result = AIMessage( content=[ {"type": "reasoning", "summary": [], "id": "rs_1"}, {"type": "text", "text": "已完成视频画面检查。", "annotations": []}, ] ) summary, analysis, events = _extract_submit_payload(result) assert summary == analysis == "已完成视频画面检查。" assert events == [] @pytest.mark.asyncio @pytest.mark.parametrize("mode", ["parsed", "raw", "pipeline"]) async def test_wrapper_structured_output_uses_responses_function_call( transport_model, monkeypatch, mode ): class Verdict(BaseModel): ok: bool model, requests, outputs = transport_model outputs.append( [ { "type": "function_call", "name": "Verdict", "call_id": "call_verdict", "id": "fc_verdict", "arguments": '{"ok":true}', "status": "completed", } ] ) wrapper = RobustChatModelWrapper(model, endpoint=ModelEndpoint(model_name="test-model")) metered = [] monkeypatch.setattr( "artemis.services.llm.record_llm_usage", lambda engine, response, **kw: metered.append(response), ) structured = wrapper.with_structured_output(Verdict, include_raw=mode == "raw") if mode == "pipeline": from langchain_core.runnables import RunnableLambda result = await (structured | RunnableLambda(lambda value: value.ok)).ainvoke("Check") assert result is True else: result = await structured.complete("Check") assert (result["parsed"] if mode == "raw" else result).ok is True if mode == "raw": assert result["parsing_error"] is None assert result["raw"].tool_calls payload = json.loads(requests[0].content) assert requests[0].url.path == "/v1/responses" assert payload["tools"][0]["name"] == "Verdict" assert payload.get("text", {}).get("format", {}).get("type") != "json_schema" from artemis.llm.usage import usage_from_message assert len(metered) == 1 assert usage_from_message(metered[0])["total_tokens"] == 13 @pytest.mark.asyncio async def test_cache_key_is_stable_for_exploration_but_separates_roles( transport_model, monkeypatch ): from types import SimpleNamespace model, requests, _ = transport_model engine = SimpleNamespace(current_session_id="run-1") ctx = SimpleNamespace(data_engine=engine, exploration_id="exploration-1") monkeypatch.setattr("artemis.services.llm.record_llm_usage", lambda *a, **k: None) wrapper = RobustChatModelWrapper( model, ctx, endpoint=ModelEndpoint(model_name="test-model"), role="operator" ) await wrapper.ainvoke([HumanMessage(content="first")]) engine.current_session_id = "resumed-run-2" await wrapper.ainvoke([HumanMessage(content="first"), HumanMessage(content="next")]) wrapper.role = "checker" await wrapper.ainvoke([HumanMessage(content="verify")]) payloads = [json.loads(r.content) for r in requests] assert payloads[0]["prompt_cache_key"] == payloads[1]["prompt_cache_key"] assert payloads[1]["prompt_cache_key"] != payloads[2]["prompt_cache_key"] assert "exploration-1" not in payloads[0]["prompt_cache_key"] assert payloads[1]["input"][:1] == payloads[0]["input"] for request, payload in zip(requests, payloads): assert request.headers["thread-id"] == payload["prompt_cache_key"] assert request.headers["session-id"] == payload["prompt_cache_key"] assert request.headers["x-client-request-id"] assert "extra_headers" not in payload assert requests[0].headers["x-client-request-id"] != requests[1].headers["x-client-request-id"] @pytest.mark.asyncio @pytest.mark.parametrize("header", ["Thread-Id", "thread_id", "SESSION-ID", "Session_id"]) async def test_caller_affinity_header_reaches_wire_as_body_key(transport_model, header): model, requests, _ = transport_model wrapper = RobustChatModelWrapper(model, endpoint=ModelEndpoint(model_name="test-model")) headers = {header: "caller-affinity"} for _ in range(2): await wrapper.ainvoke([HumanMessage(content="shared prefix")], extra_headers=headers) for request in requests: payload = json.loads(request.content) assert payload["prompt_cache_key"] == "caller-affinity" routing = {name.lower().replace("_", "-"): value for name, value in request.headers.items()} assert routing["thread-id"] == routing["session-id"] == "caller-affinity" assert "extra_headers" not in payload assert requests[0].headers["x-client-request-id"] != requests[1].headers["x-client-request-id"] assert headers == {header: "caller-affinity"} @pytest.mark.asyncio async def test_auxiliary_summary_calls_send_role_scoped_affinity_on_responses(transport_model): from types import SimpleNamespace from artemis.agents.flash.summarizer import VisualStepSummarizer from artemis.memory.chunking import StepCapsuleLens model, requests, outputs = transport_model outputs.extend( [text_output("我看到页面显示当前设置，尚未确认其他状态变化。") for _ in range(4)] ) ctx = SimpleNamespace(data_engine=SimpleNamespace(current_session_id="summary-task")) lens = StepCapsuleLens(model_name="test-model", llm=model, ctx=ctx) for text in ("first chunk", "next chunk"): await lens._invoke(model, [HumanMessage(content=text)], "test-model") visual = VisualStepSummarizer(ctx, model_name="test-model") visual._llm = model for number in (1, 2): visual._step_inputs[number] = { "step_number": number, "action_name": "press_key", "action_args": {}, "pre_img_bytes": None, "post_img_bytes": None, "exec_outcome": "dispatched", } assert await visual._attempt(number) keys = [] for request in requests: payload = json.loads(request.content) assert request.url.path == "/v1/responses" key = payload["prompt_cache_key"] assert request.headers["thread-id"] == request.headers["session-id"] == key keys.append(key) assert len(keys) == 4 and keys[0] == keys[1] and keys[2] == keys[3] and keys[0] != keys[2]