Artemis-Eapil/tests/unit/test_case_result.py

import json

import pytest

from artemis.runtime.case_result import (
    CaseTestResult,
    normalize_case_result,
    read_case_result,
    resolve_case_result,
)
from artemis.sdk.agent import attach_test_summary, resolve_trace_suffix


def outcome(status="passed", **extra):
    return {
        "task_status": "completed",
        "test_result": {
            "status": status,
            "reason": "预期与实际对比",
            "evidence": ["步骤 2 截图显示实际结果"],
        },
        **extra,
    }


@pytest.mark.parametrize("status", ["passed", "failed"])
def test_a_claim_without_evidence_is_never_a_verdict(status):
    result = normalize_case_result(
        CaseTestResult(status=status, reason="执行者说完成了", evidence=[" "])
    )
    assert result.status == "inconclusive"
    assert "证据" in result.reason


def test_completed_execution_and_failed_test_are_independent():
    result = resolve_case_result(outcome("failed"), "completed")
    assert result.status == "failed"
    returned = attach_test_summary(
        {"status": "completed", "explanation": "已完成", "test_result": {"status": "passed"}},
        outcome("failed"),
    )
    assert returned["status"] == "completed"
    assert returned["test_result"]["status"] == "failed"
    assert resolve_trace_suffix("completed", outcome("failed")) == "_TESTFAIL"


@pytest.mark.parametrize(
    ("status", "suffix"),
    [
        ("passed", "_PASS"),
        ("failed", "_TESTFAIL"),
        ("blocked", "_BLOCKED"),
        ("inconclusive", "_UNVERIFIED"),
    ],
)
def test_completed_execution_trace_names_use_the_formal_verdict(status, suffix):
    assert resolve_trace_suffix("completed", outcome(status)) == suffix
    assert resolve_trace_suffix("completed", None) == "_UNVERIFIED"


def test_skipped_review_metadata_survives_reload_without_changing_model_schema(tmp_path):
    result = CaseTestResult(
        status="inconclusive", reason="执行完成 · 未做 AI 判定", verification_skipped=True
    )
    directory = tmp_path / "task-skipped"
    directory.mkdir()
    (directory / "run_outcome.json").write_text(
        json.dumps({"task_status": "completed", "test_result": result.model_dump()}),
        encoding="utf-8",
    )
    loaded = read_case_result("task-skipped", "completed", root=tmp_path)
    assert loaded == result.model_dump()
    assert loaded["verification_skipped"] is True
    assert "verification_skipped" not in CaseTestResult.model_json_schema()["properties"]
    assert (
        "verification_skipped" not in CaseTestResult(status="blocked", reason="失败").model_dump()
    )


@pytest.mark.parametrize("status", ["failed", "cancelled", "interrupted", "skipped"])
def test_incomplete_execution_does_not_inherit_a_pass(status):
    result = resolve_case_result(outcome(), status, "环境初始化失败")
    assert result.status == "blocked"
    assert result.reason == "环境初始化失败"


def test_a_previously_confirmed_assertion_failure_survives_interruption_and_later_pass():
    data = outcome(
        tests={
            "failed": 1,
            "failed_items": [
                {
                    "kind": "assert",
                    "item_text": "预期显示成功提示",
                    "evidence": "步骤 4 正确点击后显示错误提示",
                }
            ],
        }
    )
    result = resolve_case_result(data, "interrupted", "设备掉线")
    assert result.status == "failed"
    assert result.evidence == ["步骤 4 正确点击后显示错误提示"]


def test_navigation_failure_is_not_reported_as_a_product_defect():
    result = resolve_case_result(
        {
            "tests": {
                "failed": 1,
                "failed_items": [
                    {"kind": "verify", "item_text": "打开设置", "evidence": "页面未打开"}
                ],
            }
        },
        "failed",
    )
    assert result.status == "blocked"


def test_legacy_completed_flash_without_verification_is_inconclusive():
    assert resolve_case_result({}, "completed").status == "inconclusive"
    assert resolve_case_result(None, "running") is None
    assert resolve_case_result({"tests": {"passed": 2}}, "completed").status == "passed"


def test_missing_checks_prevent_an_overall_pass():
    assert (
        resolve_case_result(outcome(tests={"passed": 1, "unchecked": 1}), "completed").status
        == "inconclusive"
    )


def test_session_reader_handles_missing_and_corrupt_outcomes_without_claiming_pass(tmp_path):
    assert read_case_result("case", "completed", root=tmp_path)["status"] == "inconclusive"
    directory = tmp_path / "case"
    directory.mkdir()
    (directory / "run_outcome.json").write_text("{", encoding="utf-8")
    assert read_case_result("case", "completed", root=tmp_path)["status"] == "inconclusive"
    (directory / "run_outcome.json").write_text(json.dumps(outcome("failed")), encoding="utf-8")
    assert read_case_result("case", "completed", root=tmp_path)["status"] == "failed"
    with pytest.raises(ValueError):
        read_case_result("../case", "completed", root=tmp_path)


def test_failure_persistence_keeps_confirmed_test_failure_and_checkpoint_evidence(
    tmp_path, monkeypatch
):
    from artemis.runtime import task_result

    monkeypatch.setattr(task_result, "TRACES_PATH", tmp_path)
    directory = tmp_path / "case"
    directory.mkdir()
    (directory / "result.json").write_text(
        json.dumps(outcome("failed", tests={"checkpoint": {"passed": False}})),
        encoding="utf-8",
    )
    (directory / "run_outcome.json").write_text(
        json.dumps(outcome("passed", tests={"passed": 1})), encoding="utf-8"
    )
    task_result.save_failure_result("case", "任务超时")
    for name in ("result.json", "run_outcome.json"):
        saved = json.loads((directory / name).read_text(encoding="utf-8"))
        assert saved["test_result"]["status"] == "failed"
        assert saved["tests"]["checkpoint"]["passed"] is False
        assert saved["test_result"]["evidence"] == ["步骤 2 截图显示实际结果"]