Artemis-Eapil/tests/unit/agents/test_flash_acceptance.py

import asyncio
import json
from unittest.mock import AsyncMock, MagicMock
from langchain_core.messages import AIMessage

import pytest

from artemis.agents.checker import flash
from artemis.agents.checker.checker import CheckReport, ReportCorrection
from artemis.context import ArtemisContext, ExecutionSetup
from artemis.runtime.case_result import CaseTestResult


@pytest.mark.asyncio
async def test_checker_corrects_report_with_evidence_without_changing_product_failure(
    tmp_path, monkeypatch
):
    from artemis.utils.notes import save_verified_note

    body = "# 双端报告\nPC 为浅色；PC 截图没有保存；安卓导航遮挡。"
    save_verified_note(tmp_path, "report", body)
    corrections = [
        ReportCorrection(
            original="PC 为浅色", replacement="PC 为深色", evidence="步骤 35 电脑截图"
        ),
        ReportCorrection(
            original="PC 截图没有保存",
            replacement="PC 截图已保存",
            evidence="步骤 35 已验证 path/hash 回执",
        ),
    ]
    check = AsyncMock(
        return_value=CheckReport(
            test_result=CaseTestResult(
                status="failed", reason="安卓导航遮挡正文", evidence=["步骤 40"]
            ),
            report_corrections=corrections,
        )
    )
    monkeypatch.setattr(flash, "run_final_check", check)
    execution = {"status": "completed", "explanation": body, "report_note_key": "report"}
    data = await flash.settle_flash_outcome(context(tmp_path), "双端视觉检查", execution)
    assert data["test_result"]["status"] == "failed"
    assert data["report_review"]["applied"] is True
    assert execution["explanation"] == (tmp_path / "notes/output.md").read_text(encoding="utf-8")
    assert execution["explanation"] == (tmp_path / "notes/report.md").read_text(encoding="utf-8")
    assert "PC 为深色" in execution["explanation"]
    assert (tmp_path / "notes/output_before_review.md").read_text(encoding="utf-8") == body
    stored = json.loads((tmp_path / "run_outcome.json").read_text(encoding="utf-8"))
    assert stored["report_review"]["corrections"][0]["evidence"] == "步骤 35 电脑截图"


def test_report_correction_requires_exact_unique_quote(tmp_path):
    from artemis.agents.checker.report_review import apply_report_review

    execution = {"explanation": "same same"}
    result = apply_report_review(
        tmp_path,
        execution,
        [ReportCorrection(original="same", replacement="changed", evidence="步骤 1")],
    )
    assert result["applied"] is False
    assert execution["explanation"] == "same same"
    assert not (tmp_path / "notes/output.md").exists()


def context(tmp_path, **setup):
    ctx = MagicMock(spec=ArtemisContext)
    ctx.execution_setup = ExecutionSetup(**setup)
    ctx.data_engine = MagicMock()
    ctx.data_engine.base_dir = tmp_path
    return ctx


@pytest.mark.asyncio
@pytest.mark.parametrize("verdict", ["passed", "failed", "inconclusive"])
async def test_flash_reuses_checker_with_original_prompt_and_final_report(
    tmp_path, monkeypatch, verdict
):
    result = CaseTestResult(
        status=verdict, reason="提示词预期与实际对比", evidence=["步骤 3 截图显示实际状态"]
    )
    check = AsyncMock(return_value=CheckReport(test_result=result))
    monkeypatch.setattr(flash, "run_final_check", check)
    ctx = context(tmp_path)
    ctx.on_execution_review = AsyncMock()

    async def review_after_capture_stops(*args, **kwargs):
        ctx.on_execution_review.assert_awaited_once()
        receipt = kwargs["report_delivery"]
        assert receipt["verified"] is True
        assert receipt["report_path"] == "notes/output.md"
        assert (tmp_path / receipt["report_path"]).read_text(encoding="utf-8") == "操作已执行"
        return CheckReport(test_result=result)

    check.side_effect = review_after_capture_stops
    execution = {"status": "completed", "explanation": "操作已执行"}
    data = await flash.settle_flash_outcome(ctx, "打开设置，预期看到应用入口", execution)
    assert check.call_count == 1
    assert check.call_args.kwargs["goal"] == "打开设置，预期看到应用入口"
    assert check.call_args.kwargs["execution_report"] == execution
    assert check.call_args.kwargs["check_items"] == []
    assert data["task_status"] == "completed"
    assert data["test_result"]["status"] == verdict
    assert (
        json.loads((tmp_path / "run_outcome.json").read_text(encoding="utf-8"))["test_result"]
        == data["test_result"]
    )
    assert ctx.run_outcome == data


@pytest.mark.asyncio
async def test_acceptance_timeout_finishes_with_unknown_verdict(tmp_path, monkeypatch):
    from artemis.llm import final_review

    async def hanging_check(*args, **kwargs):
        await asyncio.Event().wait()

    monkeypatch.setattr(flash, "run_final_check", hanging_check)
    monkeypatch.setattr(final_review, "MAX_FINAL_REVIEW_SECONDS", 0.01)
    data = await flash.settle_flash_outcome(
        context(tmp_path, checkpoint_timeout=0.01), "任务", {"status": "completed"}
    )
    assert data["task_status"] == "completed"
    assert data["test_result"]["status"] == "inconclusive"
    assert "总预算已用尽（0.01 秒）" in data["test_result"]["reason"]


@pytest.mark.asyncio
async def test_flash_final_review_can_outlast_midway_checkpoint_timeout(tmp_path, monkeypatch):
    async def slower_review(*args, **kwargs):
        await asyncio.sleep(0.02)
        return CheckReport(
            test_result=CaseTestResult(
                status="passed", reason="完整核对实际证据", evidence=["步骤 1 截图"]
            )
        )

    monkeypatch.setattr(flash, "run_final_check", slower_review)
    data = await flash.settle_flash_outcome(
        context(tmp_path, checkpoint_timeout=0.001), "任务", {"status": "completed"}
    )
    assert data["test_result"]["status"] == "passed"


@pytest.mark.asyncio
@pytest.mark.parametrize("setup", [{"disable_checker": True}, {"disable_final_check": True}])
async def test_disabled_acceptance_does_not_call_a_model_or_claim_pass(
    tmp_path, monkeypatch, setup
):
    check = AsyncMock()
    monkeypatch.setattr(flash, "run_final_check", check)
    data = await flash.settle_flash_outcome(
        context(tmp_path, **setup), "任务", {"status": "completed"}
    )
    check.assert_not_called()
    assert data["test_result"]["status"] == "inconclusive"


@pytest.mark.asyncio
async def test_report_is_saved_without_checker_or_another_model(tmp_path, monkeypatch):
    check = AsyncMock()
    monkeypatch.setattr(flash, "run_final_check", check)
    text = "# 双端报告\n安卓与电脑证据分开列出。\n## 未测\n没有验证播放。"
    await flash.settle_flash_outcome(
        context(tmp_path, disable_checker=True),
        "报告",
        {"status": "completed", "explanation": text},
    )
    assert (tmp_path / "notes/output.md").read_text(encoding="utf-8") == text
    check.assert_not_awaited()


@pytest.mark.asyncio
async def test_report_write_failure_is_visible_to_checker_not_claimed_saved(tmp_path, monkeypatch):
    (tmp_path / "notes").write_text("not a directory", encoding="utf-8")
    check = AsyncMock(
        return_value=CheckReport(
            test_result=CaseTestResult(status="blocked", reason="报告无法保存")
        )
    )
    monkeypatch.setattr(flash, "run_final_check", check)
    outcome = await flash.settle_flash_outcome(
        context(tmp_path), "报告", {"status": "completed", "explanation": "实际报告正文"}
    )
    receipt = check.call_args.kwargs["report_delivery"]
    assert receipt["ok"] is False and receipt["verified"] is False
    assert "report_path" not in receipt
    assert outcome["test_result"]["status"] == "blocked"


@pytest.mark.asyncio
async def test_execution_failure_is_blocked_without_another_model_retry(tmp_path, monkeypatch):
    check = AsyncMock()
    monkeypatch.setattr(flash, "run_final_check", check)
    data = await flash.settle_flash_outcome(
        context(tmp_path), "任务", {"status": "failed", "explanation": "设备观测失败"}
    )
    check.assert_not_called()
    assert data["test_result"]["status"] == "blocked"
    assert data["test_result"]["reason"] == "设备观测失败"


@pytest.mark.asyncio
async def test_reported_expectation_mismatch_is_checked_even_when_executor_reports_failed(
    tmp_path, monkeypatch
):
    check = AsyncMock(
        return_value=CheckReport(
            test_result=CaseTestResult(
                status="failed",
                reason="预期英文 Search settings，实际中文 搜索设置。",
                evidence=["设置首页截图和 UI 元素 1 显示 搜索设置"],
            )
        )
    )
    monkeypatch.setattr(flash, "run_final_check", check)
    data = await flash.settle_flash_outcome(
        context(tmp_path),
        "预期搜索框为英文 Search settings",
        {"status": "failed", "explanation": "已检查，实际为中文"},
        final_report_received=True,
    )
    check.assert_awaited_once()
    assert data["test_result"]["status"] == "failed"
    assert "Search settings" in data["test_result"]["reason"]


@pytest.mark.asyncio
async def test_cancellation_of_acceptance_propagates(tmp_path, monkeypatch):
    check = AsyncMock(side_effect=asyncio.CancelledError)
    monkeypatch.setattr(flash, "run_final_check", check)
    with pytest.raises(asyncio.CancelledError):
        await flash.settle_flash_outcome(context(tmp_path), "任务", {"status": "completed"})
    assert not (tmp_path / "run_outcome.json").exists()


@pytest.mark.asyncio
async def test_complete_checker_answer_is_persisted_without_reformatting(tmp_path, monkeypatch):
    from artemis.agents.checker import checker

    ctx = context(tmp_path)
    ctx.exploration_id = None
    ctx.on_execution_review = None
    result = CaseTestResult(
        status="passed",
        reason="双端布局探索和报告达标；不代表网站无缺陷。",
        evidence=["步骤 18、27：双端真实页底；报告列出待核实风险。"],
    )
    answer = AIMessage(
        content="最终裁定\n```json\n" + json.dumps({"test_result": result.model_dump()}) + "\n```"
    )
    llm = MagicMock()
    monkeypatch.setattr(checker, "get_llm", lambda **_: llm)
    monkeypatch.setattr(checker, "acomplete", AsyncMock(return_value=answer))
    monkeypatch.setattr(checker, "build_checker_tools", lambda *args: [])
    monkeypatch.setattr(checker, "_capture_final_screen", AsyncMock(return_value=(None, "footer")))
    monkeypatch.setattr(checker, "_load_steps", lambda *args: [])

    outcome = await flash.settle_flash_outcome(
        ctx, "如实报告布局风险，不要求网站无缺陷", {"status": "completed"}
    )

    llm.with_structured_output.assert_not_called()
    assert outcome["test_result"] == result.model_dump()
    saved = json.loads((tmp_path / "run_outcome.json").read_text(encoding="utf-8"))
    assert saved["test_result"] == result.model_dump()


@pytest.mark.asyncio
@pytest.mark.parametrize("execution_target", ["android", "computer"])
async def test_flash_text_delivery_reaches_checker_through_real_trace(
    tmp_path, monkeypatch, execution_target
):
    from artemis.agents.checker import checker
    from artemis.agents.flash.runner import FlashRunner
    from artemis.data_engine import engine as engine_module
    from artemis.data_engine.engine import DataEngine
    from artemis.graph.state import State
    from artemis.memory.transcript import TranscriptLedger
    from artemis.mcp.action_executor import McpActionExecutor

    monkeypatch.setattr(engine_module, "PAUSE_FILE", tmp_path / "pause")
    monkeypatch.setattr(engine_module, "_CURRENT_DATA_ENGINE", None)
    ctx = context(tmp_path, traces_path=str(tmp_path))
    ctx.device = None
    ctx.exploration_id = None
    ctx.on_execution_review = None
    ctx.execution_target = execution_target
    ctx.task_control = None
    engine = DataEngine(ctx)
    ctx.data_engine = engine
    engine.start_session("保存交付文件并验收")
    actuator = MagicMock()
    actuator.extensions.return_value = []
    executor = McpActionExecutor(ctx, actuator=actuator, agent_name="flash")
    report_body = "# 双端报告\n实际发现及未测范围。"
    verdict = CaseTestResult(status="inconclusive", reason="交付已保存，业务证据不足")

    async def inspect_request(llm, messages, **kwargs):
        text = "\n".join(block.get("text", "") for block in messages[-1].content)
        assert '"tool": "save_note"' in text
        assert '"path": "notes/findings.md"' in text and '"verified": true' in text
        assert '"report_path": "notes/output.md"' in text
        assert (engine.base_dir / "notes/output.md").read_text(encoding="utf-8") == report_body
        assert (engine.base_dir / "notes/findings.md").read_text(encoding="utf-8") == "业务观察记录"
        return AIMessage(content=json.dumps({"test_result": verdict.model_dump()}))

    model = AsyncMock(side_effect=inspect_request)
    monkeypatch.setattr(checker, "get_llm", lambda **_: MagicMock())
    monkeypatch.setattr(checker, "acomplete", model)
    monkeypatch.setattr(checker, "_capture_final_screen", AsyncMock(return_value=(None, "")))
    monkeypatch.setattr("artemis.controllers.unified_controller.get_driver", lambda *_: MagicMock())
    runner = FlashRunner(ctx, goal="核实双端布局并保存报告", max_turns=0)
    runner.executor = executor
    runner.summarizer = None
    runner._init_llm = MagicMock(return_value=MagicMock())
    runner._prepare_conversation = AsyncMock(return_value=(TranscriptLedger(), None, ""))
    runner._refresh_observation = AsyncMock(side_effect=AssertionError("note must not observe"))
    runner._android_observation_stale = True
    runner._invoke_model = AsyncMock(
        side_effect=[
            AIMessage(
                content="保存已观察结果。",
                tool_calls=[
                    {
                        "name": "save_note",
                        "args": {"key": "findings", "content": "业务观察记录"},
                        "id": "save-1",
                    }
                ],
            ),
            AIMessage(
                content="提交报告。",
                tool_calls=[
                    {
                        "name": "report_task_status",
                        "args": {"status": "completed", "explanation": report_body},
                        "id": "done",
                    }
                ],
            ),
        ]
    )
    try:
        execution = await runner.run(State(initial_goal=runner.goal))
        await engine.shutdown()
        steps = engine.get_agent_friendly_steps()
        assert steps[0]["action_taken"]["action"] == "save_note"
        assert steps[0]["tool_calls"][0]["name"] == "save_note"
        assert not steps[0].get("pre_image_name") and not steps[0].get("post_image_name")
        runner._refresh_observation.assert_not_awaited()
        outcome = await flash.settle_flash_outcome(ctx, runner.goal, execution)
        model.assert_awaited_once()
        assert outcome["test_result"] == verdict.model_dump()
        actuator.execute.assert_not_called()
    finally:
        await engine.shutdown()