feat: prepare 2.2.0 with UI and execution fixes
Add dark theme, richer Markdown and readable plugin results. Improve Android input evidence, response integrity and diagnostics. Unify lifecycle, guidance receipts and evidence sources across clients. Keep direct action dispatch without an added AI quota approval layer. Align version metadata at 2.2.0 and bundle Helper 1.2.2. Validation: 3063 backend tests before the version bump; 44 after. Frontend: 317 tests and production build passed.
This commit is contained in:
chenmingxuan committed
1 parent
83bcfd54d5
commit
fac4ccb760
146 files changed
+5689
-2033
No files matched your search
@@ -0,0 +1,276 @@
|
||||
"""Reject unfinished Responses output before parsers or tool consumers see it."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import httpx
|
||||
from langchain_core.messages import AIMessage, AIMessageChunk, HumanMessage, ToolMessage
|
||||
from langchain_core.runnables import RunnableLambda
|
||||
from pydantic import BaseModel, SecretStr
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from artemis.llm.reliability import FailureCategory, LLMExhaustedError
|
||||
from artemis.llm.response_guard import ResponseIntegrityError, ensure_response_complete
|
||||
from artemis.llm.router import ModelEndpoint, ModelFactory
|
||||
from artemis.services import llm as llm_service
|
||||
from artemis.services.llm import RobustChatModelWrapper, acomplete
|
||||
|
||||
|
||||
def response_body(status, arguments, name):
|
||||
return {
|
||||
"id": "resp_test",
|
||||
"object": "response",
|
||||
"created_at": 1,
|
||||
"status": status,
|
||||
"model": "test-model",
|
||||
"output": [
|
||||
{
|
||||
"type": "function_call",
|
||||
"id": "fc_test",
|
||||
"call_id": "call_test",
|
||||
"name": name,
|
||||
"arguments": arguments,
|
||||
"status": "completed",
|
||||
}
|
||||
],
|
||||
"incomplete_details": {"reason": "max_output_tokens"} if status == "incomplete" else None,
|
||||
"usage": {"input_tokens": 10, "output_tokens": 3, "total_tokens": 13},
|
||||
}
|
||||
|
||||
|
||||
def response_stream(status, arguments, name):
|
||||
body = response_body(status or "in_progress", arguments, name)
|
||||
events = [
|
||||
{"type": "response.created", "response": {**body, "status": "in_progress", "output": []}},
|
||||
{
|
||||
"type": "response.output_item.added",
|
||||
"output_index": 0,
|
||||
"item": {**body["output"][0], "arguments": "", "status": "in_progress"},
|
||||
},
|
||||
{
|
||||
"type": "response.function_call_arguments.delta",
|
||||
"output_index": 0,
|
||||
"item_id": "fc_test",
|
||||
"delta": arguments,
|
||||
},
|
||||
]
|
||||
if status:
|
||||
events.append({"type": "response." + status, "response": body})
|
||||
return "".join("data: " + json.dumps(event) + "\n\n" for event in events) + "data: [DONE]\n\n"
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def transport_case(monkeypatch, tmp_path):
|
||||
from langchain_openai import ChatOpenAI
|
||||
import openai
|
||||
|
||||
requests, outputs, clients, events = [], [], [], []
|
||||
|
||||
def handle(request):
|
||||
payload = json.loads(request.content)
|
||||
requests.append(payload)
|
||||
status, arguments = outputs.pop(0)
|
||||
name = payload.get("tools", [{"name": "finish"}])[0]["name"]
|
||||
if payload.get("stream"):
|
||||
return httpx.Response(
|
||||
200,
|
||||
text=response_stream(status, arguments, name),
|
||||
headers={"content-type": "text/event-stream"},
|
||||
)
|
||||
return httpx.Response(200, json=response_body(status, arguments, name))
|
||||
|
||||
def http_client(**kwargs):
|
||||
client = httpx.AsyncClient(transport=httpx.MockTransport(handle), **kwargs)
|
||||
clients.append(client)
|
||||
return client
|
||||
|
||||
monkeypatch.setattr(openai, "DefaultAsyncHttpxClient", http_client)
|
||||
monkeypatch.setattr(llm_service.settings, "OPENAI_API_KEY", SecretStr("test-secret"))
|
||||
monkeypatch.setattr(llm_service.settings, "OPENAI_BASE_URL", "https://responses.example/v1")
|
||||
monkeypatch.setattr(llm_service.settings, "OPENAI_MAX_RETRIES", 5)
|
||||
monkeypatch.setattr(llm_service.settings, "OPENAI_RETRY_INTERVAL_SECONDS", 0)
|
||||
monkeypatch.delenv("ARTEMIS_FAKE_LLM", raising=False)
|
||||
monkeypatch.setattr(llm_service, "PAUSE_FILE", tmp_path / ".artemis_paused")
|
||||
monkeypatch.setattr(llm_service, "_wait_for_resume", AsyncMock(return_value=False))
|
||||
monkeypatch.setattr(llm_service, "_get_current_data_engine", lambda: None)
|
||||
monkeypatch.setattr(
|
||||
llm_service, "_record_llm_event", lambda *args, **kwargs: events.append(args)
|
||||
)
|
||||
llm_service._ENDPOINT_BREAKER._states.clear()
|
||||
llm_service._NON_STREAMING_ENDPOINTS.clear()
|
||||
endpoint = ModelEndpoint(model_name="test-model", max_retries=0)
|
||||
model = ModelFactory.create_model(endpoint)
|
||||
assert isinstance(model, ChatOpenAI)
|
||||
yield SimpleNamespace(
|
||||
model=model, endpoint=endpoint, requests=requests, outputs=outputs, events=events
|
||||
)
|
||||
for client in clients:
|
||||
await client.aclose()
|
||||
llm_service._ENDPOINT_BREAKER._states.clear()
|
||||
llm_service._NON_STREAMING_ENDPOINTS.clear()
|
||||
|
||||
|
||||
def wrapped(case):
|
||||
return RobustChatModelWrapper(case.model, endpoint=case.endpoint).bind_tools(
|
||||
[
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "finish",
|
||||
"description": "Record the verdict",
|
||||
"parameters": {"type": "object", "properties": {"passed": {"type": "boolean"}}},
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("status", [None, "incomplete", "failed"])
|
||||
@pytest.mark.parametrize("arguments", ['{"passed":true}', '{"passed":true'])
|
||||
async def test_unfinished_stream_never_returns_a_verdict_or_tool_call(
|
||||
transport_case, status, arguments
|
||||
):
|
||||
case = transport_case
|
||||
case.outputs.append((status, arguments))
|
||||
with pytest.raises(LLMExhaustedError) as failure:
|
||||
await wrapped(case).complete([HumanMessage(content="Verify the task")])
|
||||
assert failure.value.failure.category is FailureCategory.INVALID_OUTPUT
|
||||
assert len(case.requests) == 1
|
||||
resets = [payload for event, payload in case.events if event == "llm_stream_reset"]
|
||||
assert len(resets) == 1 and resets[0]["action"] == "discard"
|
||||
assert not llm_service._NON_STREAMING_ENDPOINTS
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("arguments", ['{"passed":true', ""])
|
||||
async def test_completed_event_cannot_repair_invalid_tool_json(transport_case, arguments):
|
||||
case = transport_case
|
||||
case.outputs.append(("completed", arguments))
|
||||
with pytest.raises(LLMExhaustedError) as failure:
|
||||
await wrapped(case).complete([])
|
||||
assert failure.value.failure.category is FailureCategory.INVALID_OUTPUT
|
||||
assert len(case.requests) == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("arguments", [None, "[]", "null", '{"passed":true'])
|
||||
def test_original_function_call_arguments_must_be_a_complete_json_object(arguments):
|
||||
message = AIMessage(
|
||||
content=[{"type": "function_call", "arguments": arguments}],
|
||||
response_metadata={"model_provider": "openai", "status": "completed"},
|
||||
)
|
||||
with pytest.raises(ResponseIntegrityError, match="invalid_tool_arguments"):
|
||||
ensure_response_complete(message)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completed_stream_returns_original_valid_arguments(transport_case):
|
||||
case = transport_case
|
||||
case.outputs.append(("completed", '{"passed":true}'))
|
||||
result = await wrapped(case).complete([])
|
||||
assert result.response_metadata["status"] == "completed"
|
||||
assert result.tool_calls[0]["args"] == {"passed": True}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("recovers", [False, True])
|
||||
async def test_integrity_retry_is_bounded_and_keeps_existing_tool_evidence(
|
||||
transport_case, recovers
|
||||
):
|
||||
case = transport_case
|
||||
case.endpoint.max_retries = 5
|
||||
case.outputs.extend(
|
||||
[(None, '{"passed":false}'), ("completed" if recovers else None, '{"passed":true}')]
|
||||
)
|
||||
messages = [
|
||||
HumanMessage(content="Inspect then verify"),
|
||||
AIMessage(content="", tool_calls=[{"name": "inspect", "args": {}, "id": "prior_call"}]),
|
||||
ToolMessage(content="Existing screenshot evidence", tool_call_id="prior_call"),
|
||||
]
|
||||
if recovers:
|
||||
result = await wrapped(case).complete(messages)
|
||||
assert result.tool_calls[0]["args"] == {"passed": True}
|
||||
else:
|
||||
with pytest.raises(LLMExhaustedError):
|
||||
await wrapped(case).complete(messages)
|
||||
assert len(case.requests) == 2
|
||||
assert case.requests[0]["input"] == case.requests[1]["input"]
|
||||
assert any(
|
||||
item.get("call_id") == "prior_call" and item.get("output") == "Existing screenshot evidence"
|
||||
for item in case.requests[1]["input"]
|
||||
)
|
||||
assert isinstance(messages[-1], ToolMessage)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("status", ["incomplete", "in_progress", "queued", "cancelled"])
|
||||
async def test_non_streaming_response_also_requires_completed(transport_case, status):
|
||||
case = transport_case
|
||||
case.outputs.append((status, '{"passed":true}'))
|
||||
with pytest.raises(LLMExhaustedError) as failure:
|
||||
await wrapped(case).ainvoke([])
|
||||
assert failure.value.failure.category is FailureCategory.INVALID_OUTPUT
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("include_raw", [False, True])
|
||||
async def test_structured_output_is_checked_before_returning_parsed_verdict(
|
||||
transport_case, include_raw
|
||||
):
|
||||
class Verdict(BaseModel):
|
||||
passed: bool
|
||||
|
||||
case = transport_case
|
||||
case.outputs.append(("incomplete", '{"passed":true}'))
|
||||
model = RobustChatModelWrapper(case.model, endpoint=case.endpoint)
|
||||
with pytest.raises(LLMExhaustedError):
|
||||
await model.with_structured_output(Verdict, include_raw=include_raw).ainvoke("Verify")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_downstream_parser_never_receives_incomplete_response(transport_case):
|
||||
case = transport_case
|
||||
case.outputs.append(("incomplete", '{"passed":true}'))
|
||||
parsed = []
|
||||
chain = wrapped(case) | RunnableLambda(lambda value: parsed.append(value))
|
||||
with pytest.raises(LLMExhaustedError):
|
||||
await chain.ainvoke([])
|
||||
assert parsed == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_raw_model_compatibility_path_does_not_accept_missing_terminal_event(transport_case):
|
||||
case = transport_case
|
||||
case.outputs.append((None, '{"passed":true}'))
|
||||
with pytest.raises(ValueError):
|
||||
await acomplete(case.model, [])
|
||||
assert len(case.requests) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancellation_still_closes_stream_without_retry(transport_case):
|
||||
started, closed = asyncio.Event(), []
|
||||
|
||||
class StreamingModel:
|
||||
async def astream(self, *args, **kwargs):
|
||||
try:
|
||||
yield AIMessageChunk(
|
||||
content="partial", response_metadata={"model_provider": "openai"}
|
||||
)
|
||||
started.set()
|
||||
await asyncio.Event().wait()
|
||||
finally:
|
||||
closed.append(True)
|
||||
|
||||
task = asyncio.create_task(RobustChatModelWrapper(StreamingModel()).complete([]))
|
||||
await asyncio.wait_for(started.wait(), timeout=2)
|
||||
task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await task
|
||||
assert closed == [True]
|
||||
assert not transport_case.requests
|
||||
resets = [payload for event, payload in transport_case.events if event == "llm_stream_reset"]
|
||||
assert len(resets) == 1 and resets[0]["reason"] == "incomplete_response"
|
||||
Reference in new issue
Block a user