216 lines
7.3 KiB
Python
216 lines
7.3 KiB
Python
"""Transport-level checks: every model call must use /responses."""
|
|
|
|
import json
|
|
from unittest.mock import MagicMock
|
|
|
|
import httpx
|
|
import pytest
|
|
from langchain_core.messages import HumanMessage, ToolMessage
|
|
from pydantic import BaseModel, SecretStr, ValidationError
|
|
|
|
from artemis.config.settings import settings
|
|
from artemis.llm.router import ModelEndpoint, ModelFactory, ModelProvider
|
|
from artemis.services.llm import RobustChatModelWrapper
|
|
|
|
|
|
def response_body(output):
|
|
return {
|
|
"id": "resp_test",
|
|
"object": "response",
|
|
"created_at": 1,
|
|
"status": "completed",
|
|
"model": "test-model",
|
|
"output": output,
|
|
"usage": {"input_tokens": 10, "output_tokens": 3, "total_tokens": 13},
|
|
}
|
|
|
|
|
|
def text_output(text):
|
|
return [
|
|
{
|
|
"id": "msg_test",
|
|
"type": "message",
|
|
"role": "assistant",
|
|
"status": "completed",
|
|
"content": [{"type": "output_text", "text": text, "annotations": []}],
|
|
}
|
|
]
|
|
|
|
|
|
@pytest.fixture
|
|
def transport_model(monkeypatch):
|
|
import langchain_openai
|
|
|
|
original = langchain_openai.ChatOpenAI
|
|
requests = []
|
|
outputs = []
|
|
|
|
def handle(request):
|
|
requests.append(request)
|
|
return httpx.Response(
|
|
200, json=response_body(outputs.pop(0) if outputs else text_output("OK"))
|
|
)
|
|
|
|
client = httpx.AsyncClient(transport=httpx.MockTransport(handle))
|
|
monkeypatch.setattr(
|
|
langchain_openai,
|
|
"ChatOpenAI",
|
|
lambda **kwargs: original(**kwargs, http_async_client=client),
|
|
)
|
|
monkeypatch.setattr(settings, "OPENAI_API_KEY", SecretStr("test-secret"))
|
|
monkeypatch.setattr(settings, "OPENAI_BASE_URL", "https://responses.example/v1")
|
|
monkeypatch.delenv("ARTEMIS_FAKE_LLM", raising=False)
|
|
model = ModelFactory.create_model(ModelEndpoint(model_name="test-model", timeout_seconds=23))
|
|
return model, requests, outputs
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_text_and_image_use_responses_transport(transport_model):
|
|
model, requests, _ = transport_model
|
|
result = await model.ainvoke(
|
|
[
|
|
HumanMessage(
|
|
content=[
|
|
{"type": "text", "text": "Describe"},
|
|
{"type": "image_url", "image_url": {"url": "data:image/png;base64,AA=="}},
|
|
]
|
|
)
|
|
]
|
|
)
|
|
assert requests[0].url == "https://responses.example/v1/responses"
|
|
payload = json.loads(requests[0].content)
|
|
assert payload["store"] is False
|
|
assert payload["input"][0]["content"][1]["type"] == "input_image"
|
|
assert "messages" not in payload
|
|
assert result.usage_metadata["total_tokens"] == 13
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_function_call_roundtrip_uses_call_id(transport_model):
|
|
model, requests, outputs = transport_model
|
|
outputs.append(
|
|
[
|
|
{
|
|
"type": "function_call",
|
|
"name": "lookup",
|
|
"call_id": "call_1",
|
|
"id": "fc_1",
|
|
"arguments": '{"query":"hello"}',
|
|
"status": "completed",
|
|
}
|
|
]
|
|
)
|
|
tool = {
|
|
"type": "function",
|
|
"function": {
|
|
"name": "lookup",
|
|
"description": "Look up a value",
|
|
"parameters": {
|
|
"type": "object",
|
|
"properties": {"query": {"type": "string"}},
|
|
"required": ["query"],
|
|
},
|
|
},
|
|
}
|
|
messages = [HumanMessage(content="Look up hello")]
|
|
answer = await model.bind_tools([tool]).ainvoke(messages)
|
|
assert answer.tool_calls[0]["id"] == "call_1"
|
|
await model.ainvoke([*messages, answer, ToolMessage(content="found", tool_call_id="call_1")])
|
|
payload = json.loads(requests[1].content)
|
|
assert any(
|
|
item.get("type") == "function_call_output" and item["call_id"] == "call_1"
|
|
for item in payload["input"]
|
|
)
|
|
assert all(r.url.path == "/v1/responses" for r in requests)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_structured_output_uses_responses_text_format(transport_model):
|
|
class Verdict(BaseModel):
|
|
ok: bool
|
|
|
|
model, requests, outputs = transport_model
|
|
outputs.append(text_output('{"ok":true}'))
|
|
result = await model.with_structured_output(Verdict, method="json_schema", strict=True).ainvoke(
|
|
"Check"
|
|
)
|
|
assert result.ok
|
|
payload = json.loads(requests[0].content)
|
|
assert payload["text"]["format"]["type"] == "json_schema"
|
|
assert "response_format" not in payload
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"provider", ["google", "gemini", "anthropic", "openrouter", "ollama", "custom"]
|
|
)
|
|
def test_other_provider_transports_are_rejected(provider):
|
|
with pytest.raises(ValueError, match="only OpenAI Responses"):
|
|
ModelProvider.from_string(provider)
|
|
with pytest.raises(ValidationError):
|
|
ModelEndpoint(provider=provider)
|
|
|
|
|
|
@pytest.mark.parametrize("max_retries", [0, 3])
|
|
def test_client_settings_and_cache_credentials(monkeypatch, max_retries):
|
|
monkeypatch.setattr(settings, "OPENAI_API_KEY", SecretStr("first"))
|
|
monkeypatch.setattr(settings, "OPENAI_MAX_RETRIES", max_retries)
|
|
endpoint = ModelEndpoint(api_key="test", model_name="gpt-5-mini", reasoning_effort="low")
|
|
model = ModelFactory.create_model(endpoint)
|
|
assert model.use_responses_api is True
|
|
assert model.max_retries == 0 # Gateway owns retries, never multiply at SDK layer.
|
|
assert model.reasoning_effort == "low"
|
|
dynamic = ModelEndpoint()
|
|
key = dynamic.cache_key()
|
|
monkeypatch.setattr(settings, "OPENAI_API_KEY", SecretStr("second"))
|
|
assert dynamic.cache_key() != key
|
|
assert "first" not in repr(key)
|
|
|
|
|
|
def test_web_search_uses_openai_tool():
|
|
base = MagicMock()
|
|
wrapper = RobustChatModelWrapper(base, endpoint=ModelEndpoint(enable_grounding=True))
|
|
wrapper.bind_tools([])
|
|
base.bind_tools.assert_called_once_with([{"type": "web_search"}])
|
|
|
|
|
|
def test_video_text_fallback_extracts_responses_blocks():
|
|
from langchain_core.messages import AIMessage
|
|
from artemis.agents.video_analyzer.universal_engine import _extract_submit_payload
|
|
|
|
result = AIMessage(
|
|
content=[
|
|
{"type": "reasoning", "summary": [], "id": "rs_1"},
|
|
{"type": "text", "text": "已完成视频画面检查。", "annotations": []},
|
|
]
|
|
)
|
|
summary, analysis, events = _extract_submit_payload(result)
|
|
assert summary == analysis == "已完成视频画面检查。"
|
|
assert events == []
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_wrapper_structured_output_uses_responses_function_call(transport_model):
|
|
class Verdict(BaseModel):
|
|
ok: bool
|
|
|
|
model, requests, outputs = transport_model
|
|
outputs.append(
|
|
[
|
|
{
|
|
"type": "function_call",
|
|
"name": "Verdict",
|
|
"call_id": "call_verdict",
|
|
"id": "fc_verdict",
|
|
"arguments": '{"ok":true}',
|
|
"status": "completed",
|
|
}
|
|
]
|
|
)
|
|
wrapper = RobustChatModelWrapper(model, endpoint=ModelEndpoint(model_name="test-model"))
|
|
result = await wrapper.with_structured_output(Verdict).ainvoke("Check")
|
|
assert result.ok is True
|
|
payload = json.loads(requests[0].content)
|
|
assert requests[0].url.path == "/v1/responses"
|
|
assert payload["tools"][0]["name"] == "Verdict"
|
|
assert payload.get("text", {}).get("format", {}).get("type") != "json_schema"
|