"""Transport-level checks: every model call must use /responses.""" import json from unittest.mock import MagicMock import httpx import pytest from langchain_core.messages import HumanMessage, SystemMessage, ToolMessage from pydantic import BaseModel, SecretStr, ValidationError from artemis.config.settings import settings from artemis.llm.messages import adapt_responses_messages from artemis.llm.router import ModelEndpoint, ModelFactory, ModelProvider from artemis.services.llm import RobustChatModelWrapper def response_body(output): return { "id": "resp_test", "object": "response", "created_at": 1, "status": "completed", "model": "test-model", "output": output, "usage": {"input_tokens": 10, "output_tokens": 3, "total_tokens": 13}, } def text_output(text): return [ { "id": "msg_test", "type": "message", "role": "assistant", "status": "completed", "content": [{"type": "output_text", "text": text, "annotations": []}], } ] @pytest.fixture def transport_model(monkeypatch): import langchain_openai original = langchain_openai.ChatOpenAI requests = [] outputs = [] def handle(request): requests.append(request) return httpx.Response( 200, json=response_body(outputs.pop(0) if outputs else text_output("OK")) ) client = httpx.AsyncClient(transport=httpx.MockTransport(handle)) monkeypatch.setattr( langchain_openai, "ChatOpenAI", lambda **kwargs: original(**kwargs, http_async_client=client), ) monkeypatch.setattr(settings, "OPENAI_API_KEY", SecretStr("test-secret")) monkeypatch.setattr(settings, "OPENAI_BASE_URL", "https://responses.example/v1") monkeypatch.delenv("ARTEMIS_FAKE_LLM", raising=False) model = ModelFactory.create_model(ModelEndpoint(model_name="test-model", timeout_seconds=23)) return model, requests, outputs @pytest.mark.asyncio async def test_text_and_image_use_responses_transport(transport_model): model, requests, _ = transport_model result = await model.ainvoke( [ HumanMessage( content=[ {"type": "text", "text": "Describe"}, {"type": "image_url", "image_url": {"url": "data:image/png;base64,AA=="}}, ] ) ] ) assert requests[0].url == "https://responses.example/v1/responses" payload = json.loads(requests[0].content) assert payload["store"] is False assert payload["input"][0]["content"][1]["type"] == "input_image" assert "messages" not in payload assert result.usage_metadata["total_tokens"] == 13 @pytest.mark.asyncio @pytest.mark.parametrize( ("model_name", "reasoning_effort"), [ ("mimo-v2.6-pro", "high"), ("mimo-v2.6-flash", "high"), ("gpt-6-luna", "max"), ("gpt-6-luna", "high"), ], ) async def test_selected_models_put_instructions_in_user_input( transport_model, model_name, reasoning_effort ): _, requests, _ = transport_model endpoint = ModelEndpoint( model_name=model_name, reasoning_effort=reasoning_effort, timeout_seconds=23, max_retries=0, ) model = ModelFactory.create_model(endpoint) wrapper = RobustChatModelWrapper(model, endpoint=endpoint) tool = { "type": "function", "function": { "name": "inspect_screen", "description": "Inspect the current screen", "parameters": {"type": "object", "properties": {}}, }, } await wrapper.bind_tools([tool]).ainvoke( [ SystemMessage(content="Use the available tools."), HumanMessage( content=[ {"type": "text", "text": "Inspect this screen."}, {"type": "image_url", "image_url": {"url": "data:image/png;base64,AA=="}}, ] ), ] ) payload = json.loads(requests[-1].content) assert payload["model"] == model_name assert payload["reasoning"]["effort"] == reasoning_effort assert payload["input"][0]["role"] == "user" assert payload["input"][0]["content"][0]["text"] == "Use the available tools." assert payload["input"][0]["content"][2]["type"] == "input_image" assert payload["tools"][0]["name"] == "inspect_screen" assert all(item.get("role") != "system" for item in payload["input"]) @pytest.mark.parametrize("model_name", ["gpt-6-luna", "GPT-5.6-luna"]) def test_gpt_message_adaptation_keeps_history_and_images(model_name): image = {"type": "image_url", "image_url": {"url": "data:image/png;base64,AA=="}} original = [ SystemMessage(content="First instruction"), SystemMessage(content="Second instruction"), HumanMessage(content=[{"type": "text", "text": "Observation"}, image]), SystemMessage(content="Follow-up instruction"), ] adapted = adapt_responses_messages(original, model_name) assert len(adapted) == 2 assert isinstance(adapted[0], HumanMessage) assert adapted[0].content[0]["text"] == "First instruction" assert adapted[0].content[1]["text"] == "Second instruction" assert adapted[0].content[3] == image assert isinstance(adapted[1], HumanMessage) assert adapted[1].content == [{"type": "text", "text": "Follow-up instruction"}] assert isinstance(original[0], SystemMessage) assert len(original[2].content) == 2 @pytest.mark.parametrize("model_name", ["mimo-v2.6-pro", "mimo-v2.6-flash"]) def test_mimo_messages_use_same_user_role_format(model_name): messages = [SystemMessage(content="Instructions"), HumanMessage(content="Task")] adapted = adapt_responses_messages(messages, model_name) assert len(adapted) == 1 assert isinstance(adapted[0], HumanMessage) assert adapted[0].content[0]["text"] == "Instructions" assert adapted[0].content[1]["text"] == "Task" assert isinstance(messages[0], SystemMessage) @pytest.mark.asyncio async def test_max_reasoning_is_sent_unchanged_to_responses(transport_model): model, requests, _ = transport_model model.reasoning_effort = "max" await model.ainvoke([HumanMessage(content="Check")]) assert requests[0].url.path == "/v1/responses" assert json.loads(requests[0].content)["reasoning"]["effort"] == "max" @pytest.mark.asyncio async def test_function_call_roundtrip_uses_call_id(transport_model): model, requests, outputs = transport_model outputs.append( [ { "type": "function_call", "name": "lookup", "call_id": "call_1", "id": "fc_1", "arguments": '{"query":"hello"}', "status": "completed", } ] ) tool = { "type": "function", "function": { "name": "lookup", "description": "Look up a value", "parameters": { "type": "object", "properties": {"query": {"type": "string"}}, "required": ["query"], }, }, } messages = [HumanMessage(content="Look up hello")] answer = await model.bind_tools([tool]).ainvoke(messages) assert answer.tool_calls[0]["id"] == "call_1" await model.ainvoke([*messages, answer, ToolMessage(content="found", tool_call_id="call_1")]) payload = json.loads(requests[1].content) assert any( item.get("type") == "function_call_output" and item["call_id"] == "call_1" for item in payload["input"] ) assert all(r.url.path == "/v1/responses" for r in requests) @pytest.mark.asyncio async def test_structured_output_uses_responses_text_format(transport_model): class Verdict(BaseModel): ok: bool model, requests, outputs = transport_model outputs.append(text_output('{"ok":true}')) result = await model.with_structured_output(Verdict, method="json_schema", strict=True).ainvoke( "Check" ) assert result.ok payload = json.loads(requests[0].content) assert payload["text"]["format"]["type"] == "json_schema" assert "response_format" not in payload @pytest.mark.parametrize( "provider", ["google", "gemini", "anthropic", "openrouter", "ollama", "custom"] ) def test_other_provider_transports_are_rejected(provider): with pytest.raises(ValueError, match="only OpenAI Responses"): ModelProvider.from_string(provider) with pytest.raises(ValidationError): ModelEndpoint(provider=provider) @pytest.mark.parametrize("max_retries", [0, 3]) def test_client_settings_and_cache_credentials(monkeypatch, max_retries): monkeypatch.setattr(settings, "OPENAI_API_KEY", SecretStr("first")) monkeypatch.setattr(settings, "OPENAI_MAX_RETRIES", max_retries) endpoint = ModelEndpoint(api_key="test", model_name="gpt-5-mini", reasoning_effort="low") model = ModelFactory.create_model(endpoint) assert model.use_responses_api is True assert model.max_retries == 0 # Gateway owns retries, never multiply at SDK layer. assert model.reasoning_effort == "low" dynamic = ModelEndpoint() key = dynamic.cache_key() monkeypatch.setattr(settings, "OPENAI_API_KEY", SecretStr("second")) assert dynamic.cache_key() != key assert "first" not in repr(key) def test_web_search_uses_openai_tool(): base = MagicMock() wrapper = RobustChatModelWrapper(base, endpoint=ModelEndpoint(enable_grounding=True)) wrapper.bind_tools([]) base.bind_tools.assert_called_once_with([{"type": "web_search"}]) def test_video_text_fallback_extracts_responses_blocks(): from langchain_core.messages import AIMessage from artemis.agents.video_analyzer.universal_engine import _extract_submit_payload result = AIMessage( content=[ {"type": "reasoning", "summary": [], "id": "rs_1"}, {"type": "text", "text": "已完成视频画面检查。", "annotations": []}, ] ) summary, analysis, events = _extract_submit_payload(result) assert summary == analysis == "已完成视频画面检查。" assert events == [] @pytest.mark.asyncio async def test_wrapper_structured_output_uses_responses_function_call(transport_model): class Verdict(BaseModel): ok: bool model, requests, outputs = transport_model outputs.append( [ { "type": "function_call", "name": "Verdict", "call_id": "call_verdict", "id": "fc_verdict", "arguments": '{"ok":true}', "status": "completed", } ] ) wrapper = RobustChatModelWrapper(model, endpoint=ModelEndpoint(model_name="test-model")) result = await wrapper.with_structured_output(Verdict).ainvoke("Check") assert result.ok is True payload = json.loads(requests[0].content) assert requests[0].url.path == "/v1/responses" assert payload["tools"][0]["name"] == "Verdict" assert payload.get("text", {}).get("format", {}).get("type") != "json_schema"