import asyncio import json from unittest.mock import AsyncMock, MagicMock from langchain_core.messages import AIMessage import pytest from artemis.agents.checker import flash from artemis.agents.checker.checker import CheckReport, ReportCorrection from artemis.context import ArtemisContext, ExecutionSetup from artemis.runtime.case_result import CaseTestResult @pytest.mark.asyncio async def test_checker_corrects_report_with_evidence_without_changing_product_failure( tmp_path, monkeypatch ): from artemis.utils.notes import save_verified_note body = "# 双端报告\nPC 为浅色;PC 截图没有保存;安卓导航遮挡。" save_verified_note(tmp_path, "report", body) corrections = [ ReportCorrection( original="PC 为浅色", replacement="PC 为深色", evidence="步骤 35 电脑截图" ), ReportCorrection( original="PC 截图没有保存", replacement="PC 截图已保存", evidence="步骤 35 已验证 path/hash 回执", ), ] check = AsyncMock( return_value=CheckReport( test_result=CaseTestResult( status="failed", reason="安卓导航遮挡正文", evidence=["步骤 40"] ), report_corrections=corrections, ) ) monkeypatch.setattr(flash, "run_final_check", check) execution = {"status": "completed", "explanation": body, "report_note_key": "report"} data = await flash.settle_flash_outcome(context(tmp_path), "双端视觉检查", execution) assert data["test_result"]["status"] == "failed" assert data["report_review"]["applied"] is True assert execution["explanation"] == (tmp_path / "notes/output.md").read_text(encoding="utf-8") assert execution["explanation"] == (tmp_path / "notes/report.md").read_text(encoding="utf-8") assert "PC 为深色" in execution["explanation"] assert (tmp_path / "notes/output_before_review.md").read_text(encoding="utf-8") == body stored = json.loads((tmp_path / "run_outcome.json").read_text(encoding="utf-8")) assert stored["report_review"]["corrections"][0]["evidence"] == "步骤 35 电脑截图" def test_report_correction_requires_exact_unique_quote(tmp_path): from artemis.agents.checker.report_review import apply_report_review execution = {"explanation": "same same"} result = apply_report_review( tmp_path, execution, [ReportCorrection(original="same", replacement="changed", evidence="步骤 1")], ) assert result["applied"] is False assert execution["explanation"] == "same same" assert not (tmp_path / "notes/output.md").exists() def context(tmp_path, **setup): ctx = MagicMock(spec=ArtemisContext) ctx.execution_setup = ExecutionSetup(**setup) ctx.data_engine = MagicMock() ctx.data_engine.base_dir = tmp_path return ctx @pytest.mark.asyncio @pytest.mark.parametrize("verdict", ["passed", "failed", "inconclusive"]) async def test_flash_reuses_checker_with_original_prompt_and_final_report( tmp_path, monkeypatch, verdict ): result = CaseTestResult( status=verdict, reason="提示词预期与实际对比", evidence=["步骤 3 截图显示实际状态"] ) check = AsyncMock(return_value=CheckReport(test_result=result)) monkeypatch.setattr(flash, "run_final_check", check) ctx = context(tmp_path) ctx.on_execution_review = AsyncMock() async def review_after_capture_stops(*args, **kwargs): ctx.on_execution_review.assert_awaited_once() receipt = kwargs["report_delivery"] assert receipt["verified"] is True assert receipt["report_path"] == "notes/output.md" assert (tmp_path / receipt["report_path"]).read_text(encoding="utf-8") == "操作已执行" return CheckReport(test_result=result) check.side_effect = review_after_capture_stops execution = {"status": "completed", "explanation": "操作已执行"} data = await flash.settle_flash_outcome(ctx, "打开设置,预期看到应用入口", execution) assert check.call_count == 1 assert check.call_args.kwargs["goal"] == "打开设置,预期看到应用入口" assert check.call_args.kwargs["execution_report"] == execution assert check.call_args.kwargs["check_items"] == [] assert data["task_status"] == "completed" assert data["test_result"]["status"] == verdict assert ( json.loads((tmp_path / "run_outcome.json").read_text(encoding="utf-8"))["test_result"] == data["test_result"] ) assert ctx.run_outcome == data @pytest.mark.asyncio async def test_acceptance_timeout_finishes_with_unknown_verdict(tmp_path, monkeypatch): from artemis.llm import final_review async def hanging_check(*args, **kwargs): await asyncio.Event().wait() monkeypatch.setattr(flash, "run_final_check", hanging_check) monkeypatch.setattr(final_review, "MAX_FINAL_REVIEW_SECONDS", 0.01) data = await flash.settle_flash_outcome( context(tmp_path, checkpoint_timeout=0.01), "任务", {"status": "completed"} ) assert data["task_status"] == "completed" assert data["test_result"]["status"] == "inconclusive" assert "总预算已用尽(0.01 秒)" in data["test_result"]["reason"] @pytest.mark.asyncio async def test_flash_final_review_can_outlast_midway_checkpoint_timeout(tmp_path, monkeypatch): async def slower_review(*args, **kwargs): await asyncio.sleep(0.02) return CheckReport( test_result=CaseTestResult( status="passed", reason="完整核对实际证据", evidence=["步骤 1 截图"] ) ) monkeypatch.setattr(flash, "run_final_check", slower_review) data = await flash.settle_flash_outcome( context(tmp_path, checkpoint_timeout=0.001), "任务", {"status": "completed"} ) assert data["test_result"]["status"] == "passed" @pytest.mark.asyncio @pytest.mark.parametrize("setup", [{"disable_checker": True}, {"disable_final_check": True}]) async def test_disabled_acceptance_does_not_call_a_model_or_claim_pass( tmp_path, monkeypatch, setup ): check = AsyncMock() monkeypatch.setattr(flash, "run_final_check", check) data = await flash.settle_flash_outcome( context(tmp_path, **setup), "任务", {"status": "completed"} ) check.assert_not_called() assert data["test_result"]["status"] == "inconclusive" @pytest.mark.asyncio async def test_report_is_saved_without_checker_or_another_model(tmp_path, monkeypatch): check = AsyncMock() monkeypatch.setattr(flash, "run_final_check", check) text = "# 双端报告\n安卓与电脑证据分开列出。\n## 未测\n没有验证播放。" await flash.settle_flash_outcome( context(tmp_path, disable_checker=True), "报告", {"status": "completed", "explanation": text}, ) assert (tmp_path / "notes/output.md").read_text(encoding="utf-8") == text check.assert_not_awaited() @pytest.mark.asyncio async def test_report_write_failure_is_visible_to_checker_not_claimed_saved(tmp_path, monkeypatch): (tmp_path / "notes").write_text("not a directory", encoding="utf-8") check = AsyncMock( return_value=CheckReport( test_result=CaseTestResult(status="blocked", reason="报告无法保存") ) ) monkeypatch.setattr(flash, "run_final_check", check) outcome = await flash.settle_flash_outcome( context(tmp_path), "报告", {"status": "completed", "explanation": "实际报告正文"} ) receipt = check.call_args.kwargs["report_delivery"] assert receipt["ok"] is False and receipt["verified"] is False assert "report_path" not in receipt assert outcome["test_result"]["status"] == "blocked" @pytest.mark.asyncio async def test_execution_failure_is_blocked_without_another_model_retry(tmp_path, monkeypatch): check = AsyncMock() monkeypatch.setattr(flash, "run_final_check", check) data = await flash.settle_flash_outcome( context(tmp_path), "任务", {"status": "failed", "explanation": "设备观测失败"} ) check.assert_not_called() assert data["test_result"]["status"] == "blocked" assert data["test_result"]["reason"] == "设备观测失败" @pytest.mark.asyncio async def test_reported_expectation_mismatch_is_checked_even_when_executor_reports_failed( tmp_path, monkeypatch ): check = AsyncMock( return_value=CheckReport( test_result=CaseTestResult( status="failed", reason="预期英文 Search settings,实际中文 搜索设置。", evidence=["设置首页截图和 UI 元素 1 显示 搜索设置"], ) ) ) monkeypatch.setattr(flash, "run_final_check", check) data = await flash.settle_flash_outcome( context(tmp_path), "预期搜索框为英文 Search settings", {"status": "failed", "explanation": "已检查,实际为中文"}, final_report_received=True, ) check.assert_awaited_once() assert data["test_result"]["status"] == "failed" assert "Search settings" in data["test_result"]["reason"] @pytest.mark.asyncio async def test_cancellation_of_acceptance_propagates(tmp_path, monkeypatch): check = AsyncMock(side_effect=asyncio.CancelledError) monkeypatch.setattr(flash, "run_final_check", check) with pytest.raises(asyncio.CancelledError): await flash.settle_flash_outcome(context(tmp_path), "任务", {"status": "completed"}) assert not (tmp_path / "run_outcome.json").exists() @pytest.mark.asyncio async def test_complete_checker_answer_is_persisted_without_reformatting(tmp_path, monkeypatch): from artemis.agents.checker import checker ctx = context(tmp_path) ctx.exploration_id = None ctx.on_execution_review = None result = CaseTestResult( status="passed", reason="双端布局探索和报告达标;不代表网站无缺陷。", evidence=["步骤 18、27:双端真实页底;报告列出待核实风险。"], ) answer = AIMessage( content="最终裁定\n```json\n" + json.dumps({"test_result": result.model_dump()}) + "\n```" ) llm = MagicMock() monkeypatch.setattr(checker, "get_llm", lambda **_: llm) monkeypatch.setattr(checker, "acomplete", AsyncMock(return_value=answer)) monkeypatch.setattr(checker, "build_checker_tools", lambda *args: []) monkeypatch.setattr(checker, "_capture_final_screen", AsyncMock(return_value=(None, "footer"))) monkeypatch.setattr(checker, "_load_steps", lambda *args: []) outcome = await flash.settle_flash_outcome( ctx, "如实报告布局风险,不要求网站无缺陷", {"status": "completed"} ) llm.with_structured_output.assert_not_called() assert outcome["test_result"] == result.model_dump() saved = json.loads((tmp_path / "run_outcome.json").read_text(encoding="utf-8")) assert saved["test_result"] == result.model_dump() @pytest.mark.asyncio @pytest.mark.parametrize("execution_target", ["android", "computer"]) async def test_flash_text_delivery_reaches_checker_through_real_trace( tmp_path, monkeypatch, execution_target ): from artemis.agents.checker import checker from artemis.agents.flash.runner import FlashRunner from artemis.data_engine import engine as engine_module from artemis.data_engine.engine import DataEngine from artemis.graph.state import State from artemis.memory.transcript import TranscriptLedger from artemis.mcp.action_executor import McpActionExecutor monkeypatch.setattr(engine_module, "PAUSE_FILE", tmp_path / "pause") monkeypatch.setattr(engine_module, "_CURRENT_DATA_ENGINE", None) ctx = context(tmp_path, traces_path=str(tmp_path)) ctx.device = None ctx.exploration_id = None ctx.on_execution_review = None ctx.execution_target = execution_target ctx.task_control = None engine = DataEngine(ctx) ctx.data_engine = engine engine.start_session("保存交付文件并验收") actuator = MagicMock() actuator.extensions.return_value = [] executor = McpActionExecutor(ctx, actuator=actuator, agent_name="flash") report_body = "# 双端报告\n实际发现及未测范围。" verdict = CaseTestResult(status="inconclusive", reason="交付已保存,业务证据不足") async def inspect_request(llm, messages, **kwargs): text = "\n".join(block.get("text", "") for block in messages[-1].content) assert '"tool": "save_note"' in text assert '"path": "notes/findings.md"' in text and '"verified": true' in text assert '"report_path": "notes/output.md"' in text assert (engine.base_dir / "notes/output.md").read_text(encoding="utf-8") == report_body assert (engine.base_dir / "notes/findings.md").read_text(encoding="utf-8") == "业务观察记录" return AIMessage(content=json.dumps({"test_result": verdict.model_dump()})) model = AsyncMock(side_effect=inspect_request) monkeypatch.setattr(checker, "get_llm", lambda **_: MagicMock()) monkeypatch.setattr(checker, "acomplete", model) monkeypatch.setattr(checker, "_capture_final_screen", AsyncMock(return_value=(None, ""))) monkeypatch.setattr("artemis.controllers.unified_controller.get_driver", lambda *_: MagicMock()) runner = FlashRunner(ctx, goal="核实双端布局并保存报告", max_turns=0) runner.executor = executor runner.summarizer = None runner._init_llm = MagicMock(return_value=MagicMock()) runner._prepare_conversation = AsyncMock(return_value=(TranscriptLedger(), None, "")) runner._refresh_observation = AsyncMock(side_effect=AssertionError("note must not observe")) runner._android_observation_stale = True runner._invoke_model = AsyncMock( side_effect=[ AIMessage( content="保存已观察结果。", tool_calls=[ { "name": "save_note", "args": {"key": "findings", "content": "业务观察记录"}, "id": "save-1", } ], ), AIMessage( content="提交报告。", tool_calls=[ { "name": "report_task_status", "args": {"status": "completed", "explanation": report_body}, "id": "done", } ], ), ] ) try: execution = await runner.run(State(initial_goal=runner.goal)) await engine.shutdown() steps = engine.get_agent_friendly_steps() assert steps[0]["action_taken"]["action"] == "save_note" assert steps[0]["tool_calls"][0]["name"] == "save_note" assert not steps[0].get("pre_image_name") and not steps[0].get("post_image_name") runner._refresh_observation.assert_not_awaited() outcome = await flash.settle_flash_outcome(ctx, runner.goal, execution) model.assert_awaited_once() assert outcome["test_result"] == verdict.model_dump() actuator.execute.assert_not_called() finally: await engine.shutdown()