358 lines
13 KiB
Python
358 lines
13 KiB
Python
"""Transport-level checks: every model call must use /responses."""
|
|
|
|
import json
|
|
from unittest.mock import MagicMock
|
|
|
|
import httpx
|
|
import pytest
|
|
from langchain_core.messages import HumanMessage, SystemMessage, ToolMessage
|
|
from pydantic import BaseModel, SecretStr, ValidationError
|
|
|
|
from artemis.config.settings import settings
|
|
from artemis.llm.messages import adapt_responses_messages
|
|
from artemis.llm.router import ModelEndpoint, ModelFactory, ModelProvider
|
|
from artemis.services.llm import RobustChatModelWrapper
|
|
|
|
|
|
def response_body(output):
|
|
return {
|
|
"id": "resp_test",
|
|
"object": "response",
|
|
"created_at": 1,
|
|
"status": "completed",
|
|
"model": "test-model",
|
|
"output": output,
|
|
"usage": {"input_tokens": 10, "output_tokens": 3, "total_tokens": 13},
|
|
}
|
|
|
|
|
|
def text_output(text):
|
|
return [
|
|
{
|
|
"id": "msg_test",
|
|
"type": "message",
|
|
"role": "assistant",
|
|
"status": "completed",
|
|
"content": [{"type": "output_text", "text": text, "annotations": []}],
|
|
}
|
|
]
|
|
|
|
|
|
@pytest.fixture
|
|
def transport_model(monkeypatch):
|
|
from langchain_openai import ChatOpenAI
|
|
import openai
|
|
|
|
requests = []
|
|
outputs = []
|
|
|
|
def handle(request):
|
|
requests.append(request)
|
|
return httpx.Response(
|
|
200, json=response_body(outputs.pop(0) if outputs else text_output("OK"))
|
|
)
|
|
|
|
monkeypatch.setattr(
|
|
openai,
|
|
"DefaultAsyncHttpxClient",
|
|
lambda **kwargs: httpx.AsyncClient(transport=httpx.MockTransport(handle), **kwargs),
|
|
)
|
|
monkeypatch.setattr(settings, "OPENAI_API_KEY", SecretStr("test-secret"))
|
|
monkeypatch.setattr(settings, "OPENAI_BASE_URL", "https://responses.example/v1")
|
|
monkeypatch.delenv("ARTEMIS_FAKE_LLM", raising=False)
|
|
model = ModelFactory.create_model(ModelEndpoint(model_name="test-model", timeout_seconds=23))
|
|
assert isinstance(model, ChatOpenAI)
|
|
return model, requests, outputs
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_text_and_image_use_responses_transport(transport_model):
|
|
model, requests, _ = transport_model
|
|
result = await model.ainvoke(
|
|
[
|
|
HumanMessage(
|
|
content=[
|
|
{"type": "text", "text": "Describe"},
|
|
{"type": "image_url", "image_url": {"url": "data:image/png;base64,AA=="}},
|
|
]
|
|
)
|
|
]
|
|
)
|
|
assert requests[0].url == "https://responses.example/v1/responses"
|
|
payload = json.loads(requests[0].content)
|
|
assert payload["store"] is False
|
|
assert payload["input"][0]["content"][1]["type"] == "input_image"
|
|
assert "messages" not in payload
|
|
assert result.usage_metadata["total_tokens"] == 13
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize(
|
|
("model_name", "reasoning_effort"),
|
|
[
|
|
("mimo-v2.6-pro", "high"),
|
|
("mimo-v2.6-flash", "high"),
|
|
("gpt-6-luna", "max"),
|
|
("gpt-6-luna", "high"),
|
|
],
|
|
)
|
|
async def test_selected_models_put_instructions_in_user_input(
|
|
transport_model, model_name, reasoning_effort
|
|
):
|
|
_, requests, _ = transport_model
|
|
endpoint = ModelEndpoint(
|
|
model_name=model_name,
|
|
reasoning_effort=reasoning_effort,
|
|
timeout_seconds=23,
|
|
max_retries=0,
|
|
)
|
|
model = ModelFactory.create_model(endpoint)
|
|
wrapper = RobustChatModelWrapper(model, endpoint=endpoint)
|
|
tool = {
|
|
"type": "function",
|
|
"function": {
|
|
"name": "inspect_screen",
|
|
"description": "Inspect the current screen",
|
|
"parameters": {"type": "object", "properties": {}},
|
|
},
|
|
}
|
|
await wrapper.bind_tools([tool]).ainvoke(
|
|
[
|
|
SystemMessage(content="Use the available tools."),
|
|
HumanMessage(
|
|
content=[
|
|
{"type": "text", "text": "Inspect this screen."},
|
|
{"type": "image_url", "image_url": {"url": "data:image/png;base64,AA=="}},
|
|
]
|
|
),
|
|
]
|
|
)
|
|
|
|
payload = json.loads(requests[-1].content)
|
|
assert payload["model"] == model_name
|
|
assert payload["reasoning"]["effort"] == reasoning_effort
|
|
assert payload["input"][0]["role"] == "user"
|
|
assert payload["input"][0]["content"][0]["text"] == "Use the available tools."
|
|
assert payload["input"][0]["content"][2]["type"] == "input_image"
|
|
assert payload["tools"][0]["name"] == "inspect_screen"
|
|
assert all(item.get("role") != "system" for item in payload["input"])
|
|
|
|
|
|
@pytest.mark.parametrize("model_name", ["gpt-6-luna", "GPT-5.6-luna"])
|
|
def test_gpt_message_adaptation_keeps_history_and_images(model_name):
|
|
image = {"type": "image_url", "image_url": {"url": "data:image/png;base64,AA=="}}
|
|
original = [
|
|
SystemMessage(content="First instruction"),
|
|
SystemMessage(content="Second instruction"),
|
|
HumanMessage(content=[{"type": "text", "text": "Observation"}, image]),
|
|
SystemMessage(content="Follow-up instruction"),
|
|
]
|
|
adapted = adapt_responses_messages(original, model_name)
|
|
|
|
assert len(adapted) == 2
|
|
assert isinstance(adapted[0], HumanMessage)
|
|
assert adapted[0].content[0]["text"] == "First instruction"
|
|
assert adapted[0].content[1]["text"] == "Second instruction"
|
|
assert adapted[0].content[3] == image
|
|
assert isinstance(adapted[1], HumanMessage)
|
|
assert adapted[1].content == [{"type": "text", "text": "Follow-up instruction"}]
|
|
assert isinstance(original[0], SystemMessage)
|
|
assert len(original[2].content) == 2
|
|
|
|
|
|
@pytest.mark.parametrize("model_name", ["mimo-v2.6-pro", "mimo-v2.6-flash"])
|
|
def test_mimo_messages_use_same_user_role_format(model_name):
|
|
messages = [SystemMessage(content="Instructions"), HumanMessage(content="Task")]
|
|
adapted = adapt_responses_messages(messages, model_name)
|
|
assert len(adapted) == 1
|
|
assert isinstance(adapted[0], HumanMessage)
|
|
assert adapted[0].content[0]["text"] == "Instructions"
|
|
assert adapted[0].content[1]["text"] == "Task"
|
|
assert isinstance(messages[0], SystemMessage)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_max_reasoning_is_sent_unchanged_to_responses(transport_model):
|
|
model, requests, _ = transport_model
|
|
model.reasoning_effort = "max"
|
|
await model.ainvoke([HumanMessage(content="Check")])
|
|
assert requests[0].url.path == "/v1/responses"
|
|
assert json.loads(requests[0].content)["reasoning"]["effort"] == "max"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_function_call_roundtrip_uses_call_id(transport_model):
|
|
model, requests, outputs = transport_model
|
|
outputs.append(
|
|
[
|
|
{
|
|
"type": "function_call",
|
|
"name": "lookup",
|
|
"call_id": "call_1",
|
|
"id": "fc_1",
|
|
"arguments": '{"query":"hello"}',
|
|
"status": "completed",
|
|
}
|
|
]
|
|
)
|
|
tool = {
|
|
"type": "function",
|
|
"function": {
|
|
"name": "lookup",
|
|
"description": "Look up a value",
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {"query": {"type": "string"}},
|
|
"required": ["query"],
|
|
},
|
|
},
|
|
}
|
|
messages = [HumanMessage(content="Look up hello")]
|
|
answer = await model.bind_tools([tool]).ainvoke(messages)
|
|
assert answer.tool_calls[0]["id"] == "call_1"
|
|
await model.ainvoke([*messages, answer, ToolMessage(content="found", tool_call_id="call_1")])
|
|
payload = json.loads(requests[1].content)
|
|
assert any(
|
|
item.get("type") == "function_call_output" and item["call_id"] == "call_1"
|
|
for item in payload["input"]
|
|
)
|
|
assert all(r.url.path == "/v1/responses" for r in requests)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_structured_output_uses_responses_text_format(transport_model):
|
|
class Verdict(BaseModel):
|
|
ok: bool
|
|
|
|
model, requests, outputs = transport_model
|
|
outputs.append(text_output('{"ok":true}'))
|
|
result = await model.with_structured_output(Verdict, method="json_schema", strict=True).ainvoke(
|
|
"Check"
|
|
)
|
|
assert result.ok
|
|
payload = json.loads(requests[0].content)
|
|
assert payload["text"]["format"]["type"] == "json_schema"
|
|
assert "response_format" not in payload
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"provider", ["google", "gemini", "anthropic", "openrouter", "ollama", "custom"]
|
|
)
|
|
def test_other_provider_transports_are_rejected(provider):
|
|
with pytest.raises(ValueError, match="only OpenAI Responses"):
|
|
ModelProvider.from_string(provider)
|
|
with pytest.raises(ValidationError):
|
|
ModelEndpoint(provider=provider)
|
|
|
|
|
|
@pytest.mark.parametrize("max_retries", [0, 3])
|
|
def test_client_settings_and_cache_credentials(monkeypatch, max_retries):
|
|
monkeypatch.setattr(settings, "OPENAI_API_KEY", SecretStr("first"))
|
|
monkeypatch.setattr(settings, "OPENAI_MAX_RETRIES", max_retries)
|
|
endpoint = ModelEndpoint(api_key="test", model_name="gpt-5-mini", reasoning_effort="low")
|
|
model = ModelFactory.create_model(endpoint)
|
|
assert model.use_responses_api is True
|
|
assert model.max_retries == 0 # Gateway owns retries, never multiply at SDK layer.
|
|
assert model.reasoning_effort == "low"
|
|
dynamic = ModelEndpoint()
|
|
key = dynamic.cache_key()
|
|
monkeypatch.setattr(settings, "OPENAI_API_KEY", SecretStr("second"))
|
|
assert dynamic.cache_key() != key
|
|
assert "first" not in repr(key)
|
|
|
|
|
|
def test_web_search_uses_openai_tool():
|
|
base = MagicMock()
|
|
wrapper = RobustChatModelWrapper(base, endpoint=ModelEndpoint(enable_grounding=True))
|
|
wrapper.bind_tools([])
|
|
base.bind_tools.assert_called_once_with([{"type": "web_search"}])
|
|
|
|
|
|
def test_video_text_fallback_extracts_responses_blocks():
|
|
from langchain_core.messages import AIMessage
|
|
from artemis.agents.video_analyzer.universal_engine import _extract_submit_payload
|
|
|
|
result = AIMessage(
|
|
content=[
|
|
{"type": "reasoning", "summary": [], "id": "rs_1"},
|
|
{"type": "text", "text": "已完成视频画面检查。", "annotations": []},
|
|
]
|
|
)
|
|
summary, analysis, events = _extract_submit_payload(result)
|
|
assert summary == analysis == "已完成视频画面检查。"
|
|
assert events == []
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("mode", ["parsed", "raw", "pipeline"])
|
|
async def test_wrapper_structured_output_uses_responses_function_call(
|
|
transport_model, monkeypatch, mode
|
|
):
|
|
class Verdict(BaseModel):
|
|
ok: bool
|
|
|
|
model, requests, outputs = transport_model
|
|
outputs.append(
|
|
[
|
|
{
|
|
"type": "function_call",
|
|
"name": "Verdict",
|
|
"call_id": "call_verdict",
|
|
"id": "fc_verdict",
|
|
"arguments": '{"ok":true}',
|
|
"status": "completed",
|
|
}
|
|
]
|
|
)
|
|
wrapper = RobustChatModelWrapper(model, endpoint=ModelEndpoint(model_name="test-model"))
|
|
metered = []
|
|
monkeypatch.setattr(
|
|
"artemis.services.llm.record_llm_usage",
|
|
lambda engine, response, **kw: metered.append(response),
|
|
)
|
|
structured = wrapper.with_structured_output(Verdict, include_raw=mode == "raw")
|
|
if mode == "pipeline":
|
|
from langchain_core.runnables import RunnableLambda
|
|
|
|
result = await (structured | RunnableLambda(lambda value: value.ok)).ainvoke("Check")
|
|
assert result is True
|
|
else:
|
|
result = await structured.complete("Check")
|
|
assert (result["parsed"] if mode == "raw" else result).ok is True
|
|
if mode == "raw":
|
|
assert result["parsing_error"] is None
|
|
assert result["raw"].tool_calls
|
|
payload = json.loads(requests[0].content)
|
|
assert requests[0].url.path == "/v1/responses"
|
|
assert payload["tools"][0]["name"] == "Verdict"
|
|
assert payload.get("text", {}).get("format", {}).get("type") != "json_schema"
|
|
from artemis.llm.usage import usage_from_message
|
|
|
|
assert len(metered) == 1
|
|
assert usage_from_message(metered[0])["total_tokens"] == 13
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_cache_key_is_stable_for_exploration_but_separates_roles(
|
|
transport_model, monkeypatch
|
|
):
|
|
from types import SimpleNamespace
|
|
|
|
model, requests, _ = transport_model
|
|
engine = SimpleNamespace(current_session_id="run-1")
|
|
ctx = SimpleNamespace(data_engine=engine, exploration_id="exploration-1")
|
|
monkeypatch.setattr("artemis.services.llm.record_llm_usage", lambda *a, **k: None)
|
|
wrapper = RobustChatModelWrapper(
|
|
model, ctx, endpoint=ModelEndpoint(model_name="test-model"), role="operator"
|
|
)
|
|
await wrapper.ainvoke([HumanMessage(content="first")])
|
|
engine.current_session_id = "resumed-run-2"
|
|
await wrapper.ainvoke([HumanMessage(content="first"), HumanMessage(content="next")])
|
|
wrapper.role = "checker"
|
|
await wrapper.ainvoke([HumanMessage(content="verify")])
|
|
payloads = [json.loads(r.content) for r in requests]
|
|
assert payloads[0]["prompt_cache_key"] == payloads[1]["prompt_cache_key"]
|
|
assert payloads[1]["prompt_cache_key"] != payloads[2]["prompt_cache_key"]
|
|
assert "exploration-1" not in payloads[0]["prompt_cache_key"]
|
|
assert payloads[1]["input"][:1] == payloads[0]["input"]
|