chenmingxuan
be6e8c2abd
Unify task scheduling and queue state, fix Android input and observation recovery, preserve screenshot paths, and match final review criteria accurately. Add collapsible copyable results and publish the complete 2.1.0 upgrade notes. Validation: 3189 backend tests, 326 frontend tests, 24 client tests, production build and quality gates.
237 lines
8.8 KiB
Python
237 lines
8.8 KiB
Python
# Copyright 2026 Google LLC
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""The structured verdict request must never end on a model turn (Gemini
|
|
rejects such requests with 400 "Requests ending with a model turn are not
|
|
supported"), which is the natural state after the loop's tool-free reply."""
|
|
|
|
import json
|
|
from types import SimpleNamespace
|
|
from unittest.mock import AsyncMock, MagicMock
|
|
|
|
import pytest
|
|
from langchain_core.messages import AIMessage, HumanMessage, SystemMessage
|
|
from pydantic import ValidationError
|
|
|
|
from artemis.agents.checker.checker import CheckReport, _structured_report
|
|
|
|
|
|
def _llm(report: CheckReport):
|
|
structured = MagicMock()
|
|
structured.ainvoke = AsyncMock(return_value=report)
|
|
llm = MagicMock()
|
|
llm.with_structured_output.return_value = structured
|
|
return llm, structured
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_structured_report_appends_user_turn_after_model_reply():
|
|
llm, structured = _llm(CheckReport(verdicts=[]))
|
|
messages = [SystemMessage(content="s"), HumanMessage(content="h"), AIMessage(content="done")]
|
|
|
|
await _structured_report(llm, messages)
|
|
|
|
sent = structured.ainvoke.call_args.args[0]
|
|
assert isinstance(sent[-1], HumanMessage)
|
|
assert sent[-2] is messages[-1]
|
|
assert len(messages) == 3 # caller's list untouched
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_structured_report_keeps_user_ending_conversation_as_is():
|
|
llm, structured = _llm(CheckReport(verdicts=[]))
|
|
messages = [SystemMessage(content="s"), HumanMessage(content="final iteration")]
|
|
|
|
await _structured_report(llm, messages)
|
|
|
|
assert structured.ainvoke.call_args.args[0] is messages
|
|
|
|
|
|
def _result(status="passed", evidence=None):
|
|
return {
|
|
"status": status,
|
|
"reason": "双端检查与报告已完成，已知风险不代表所有布局无缺陷。",
|
|
"evidence": ["步骤 18、27：追加下滚后位置不变，双端已到页底。"]
|
|
if evidence is None
|
|
else evidence,
|
|
}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("status", ["passed", "failed", "inconclusive", "blocked"])
|
|
async def test_complete_checker_json_does_not_request_another_verdict(status):
|
|
payload = {"test_result": _result(status)}
|
|
llm, structured = _llm(CheckReport())
|
|
answer = AIMessage(
|
|
content="核查步骤 [18]、[27]，控件 [28]。最终裁定\n```json\n"
|
|
+ json.dumps(payload)
|
|
+ "\n```\n范围仅限本次检查。"
|
|
)
|
|
|
|
result = await _structured_report(llm, [HumanMessage(content="review"), answer])
|
|
|
|
assert result.test_result.status == status
|
|
assert result.test_result.evidence == payload["test_result"]["evidence"]
|
|
structured.ainvoke.assert_not_awaited()
|
|
llm.with_structured_output.assert_not_called()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("status", ["passed", "failed", "inconclusive"])
|
|
async def test_complete_verdict_alias_and_evidence_list_are_not_sent_for_reformatting(status):
|
|
payload = {
|
|
"verdicts": [
|
|
{
|
|
"kind": "assert",
|
|
"item_text": "检查本轮页面",
|
|
"verdict": status,
|
|
"evidence": ["步骤 2 的实际页面", "步骤 3 的对照截图"],
|
|
}
|
|
],
|
|
"test_result": _result(status),
|
|
}
|
|
llm, structured = _llm(CheckReport())
|
|
report = await _structured_report(
|
|
llm,
|
|
[AIMessage(content=json.dumps(payload))],
|
|
[SimpleNamespace(kind="assert", text="检查本轮页面")],
|
|
)
|
|
assert report.verdicts[0].status == status
|
|
assert report.verdicts[0].evidence == "步骤 2 的实际页面\n步骤 3 的对照截图"
|
|
assert report.test_result.model_dump() == payload["test_result"]
|
|
structured.ainvoke.assert_not_awaited()
|
|
llm.with_structured_output.assert_not_called()
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
"fields",
|
|
[
|
|
{"status": "failed", "verdict": "passed", "evidence": ["图 1"]},
|
|
{"verdict": "unknown", "evidence": ["图 1"]},
|
|
{"verdict": "passed", "evidence": [{"text": "图 1"}]},
|
|
{"verdict": "passed", "evidence": [1]},
|
|
],
|
|
)
|
|
def test_conflicting_or_invalid_verdict_shapes_are_not_silently_accepted(fields):
|
|
with pytest.raises(ValidationError):
|
|
CheckReport.model_validate(
|
|
{"verdicts": [{"kind": "assert", "item_text": "页面", **fields}]}
|
|
)
|
|
|
|
|
|
def test_empty_evidence_list_does_not_turn_an_unsupported_failure_into_evidence():
|
|
from artemis.agents.checker.checker import _normalize_report
|
|
|
|
report = CheckReport.model_validate(
|
|
{"verdicts": [{"kind": "assert", "item_text": "页面", "verdict": "failed", "evidence": []}]}
|
|
)
|
|
assert _normalize_report(report, []).verdicts[0].status == "inconclusive"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("structured_response", [False, True])
|
|
async def test_display_labels_match_in_text_and_structured_verdict_paths(structured_response):
|
|
from artemis.agents.checker.checker import _normalize_report
|
|
from artemis.utils.plan_grammar import CheckItem
|
|
|
|
items = [
|
|
CheckItem(kind="verify", when="on_complete", text="截图保存成功", parent_key="p"),
|
|
CheckItem(kind="verify", when="on_complete", text="路径与回执一致", parent_key="p"),
|
|
CheckItem(kind="assert", when="at_end", text="没有交互动作", parent_key=None),
|
|
]
|
|
payload = {
|
|
"verdicts": [
|
|
{
|
|
"kind": i.kind,
|
|
"item_text": f"[{i.kind}] ({i.when}) {i.text}",
|
|
"status": "passed",
|
|
"evidence": "完整工具回执",
|
|
}
|
|
for i in items
|
|
],
|
|
"test_result": _result(),
|
|
}
|
|
llm, structured = _llm(CheckReport.model_validate(payload))
|
|
answer = "已检查，待结构化输出。" if structured_response else json.dumps(payload)
|
|
report = await _structured_report(llm, [AIMessage(content=answer)], items)
|
|
normalized = _normalize_report(report, items)
|
|
assert [v.item_text for v in normalized.verdicts] == [i.text for i in items]
|
|
assert [v.status for v in normalized.verdicts] == ["passed"] * 3
|
|
assert normalized.test_result.status == "passed"
|
|
if structured_response:
|
|
structured.ainvoke.assert_awaited_once()
|
|
else:
|
|
structured.ainvoke.assert_not_awaited()
|
|
|
|
|
|
def test_nested_json_string_result_is_decoded_and_still_validated():
|
|
result = _result()
|
|
assert (
|
|
CheckReport.model_validate({"test_result": json.dumps(result)}).test_result.model_dump()
|
|
== result
|
|
)
|
|
for invalid in ("passed", "null", "[]", '{"status":"passed"}', json.dumps(_result("unknown"))):
|
|
with pytest.raises(ValidationError):
|
|
CheckReport.model_validate({"test_result": invalid})
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize(
|
|
"kind", ["empty", "malformed", "truncated", "tool_call", "missing_check", "ambiguous", "prose"]
|
|
)
|
|
async def test_incomplete_or_ambiguous_answer_requires_structured_verdict(kind):
|
|
text = json.dumps({"test_result": _result()})
|
|
items = []
|
|
kwargs = {}
|
|
if kind == "empty":
|
|
text = "{}"
|
|
elif kind == "malformed":
|
|
text = '{"test_result":{"status":"passed"}}'
|
|
elif kind == "truncated":
|
|
kwargs["response_metadata"] = {"finish_reason": "length"}
|
|
elif kind == "tool_call":
|
|
kwargs["tool_calls"] = [{"id": "read-1", "name": "search_history", "args": {}}]
|
|
elif kind == "missing_check":
|
|
items = [SimpleNamespace(kind="assert", text="must pass")]
|
|
elif kind == "ambiguous":
|
|
text += "\n" + json.dumps({"test_result": _result("failed")})
|
|
elif kind == "prose":
|
|
text = "测试通过。"
|
|
llm, structured = _llm(CheckReport())
|
|
|
|
await _structured_report(llm, [AIMessage(content=text, **kwargs)], items)
|
|
|
|
structured.ainvoke.assert_awaited_once()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_local_verdict_preserves_evidence_normalization(monkeypatch):
|
|
from artemis.agents.checker import checker
|
|
|
|
llm, structured = _llm(CheckReport())
|
|
monkeypatch.setattr(checker, "get_llm", lambda **_: llm)
|
|
monkeypatch.setattr(
|
|
checker,
|
|
"acomplete",
|
|
AsyncMock(
|
|
return_value=AIMessage(
|
|
content=json.dumps({"test_result": _result(evidence=[])}),
|
|
)
|
|
),
|
|
)
|
|
ctx = SimpleNamespace(execution_setup=SimpleNamespace(checker_max_iterations=3))
|
|
report = await checker._run_check_loop(ctx, [HumanMessage(content="review")], [], [])
|
|
assert report.test_result.status == "inconclusive"
|
|
structured.ainvoke.assert_not_awaited()
|