feat: add shared test acceptance and release 1.5.0

tests/unit/test_case_result.py

@@ -0,0 +1,132 @@
+import json
+
+import pytest
+
+from artemis.runtime.case_result import (
+    CaseTestResult,
+    normalize_case_result,
+    read_case_result,
+    resolve_case_result,
+)
+from artemis.sdk.agent import attach_test_summary, resolve_trace_suffix
+
+
+def outcome(status="passed", **extra):
+    return {
+        "task_status": "completed",
+        "test_result": {
+            "status": status,
+            "reason": "预期与实际对比",
+            "evidence": ["步骤 2 截图显示实际结果"],
+        },
+        **extra,
+    }
+
+
+@pytest.mark.parametrize("status", ["passed", "failed"])
+def test_a_claim_without_evidence_is_never_a_verdict(status):
+    result = normalize_case_result(
+        CaseTestResult(status=status, reason="执行者说完成了", evidence=[" "])
+    )
+    assert result.status == "inconclusive"
+    assert "证据" in result.reason
+
+
+def test_completed_execution_and_failed_test_are_independent():
+    result = resolve_case_result(outcome("failed"), "completed")
+    assert result.status == "failed"
+    returned = attach_test_summary(
+        {"status": "completed", "explanation": "已完成", "test_result": {"status": "passed"}},
+        outcome("failed"),
+    )
+    assert returned["status"] == "completed"
+    assert returned["test_result"]["status"] == "failed"
+    assert resolve_trace_suffix("completed", outcome("failed")) == "_TESTFAIL"
+
+
+@pytest.mark.parametrize("status", ["failed", "cancelled", "interrupted", "skipped"])
+def test_incomplete_execution_does_not_inherit_a_pass(status):
+    result = resolve_case_result(outcome(), status, "环境初始化失败")
+    assert result.status == "blocked"
+    assert result.reason == "环境初始化失败"
+
+
+def test_a_previously_confirmed_assertion_failure_survives_interruption_and_later_pass():
+    data = outcome(
+        tests={
+            "failed": 1,
+            "failed_items": [
+                {
+                    "kind": "assert",
+                    "item_text": "预期显示成功提示",
+                    "evidence": "步骤 4 正确点击后显示错误提示",
+                }
+            ],
+        }
+    )
+    result = resolve_case_result(data, "interrupted", "设备掉线")
+    assert result.status == "failed"
+    assert result.evidence == ["步骤 4 正确点击后显示错误提示"]
+
+
+def test_navigation_failure_is_not_reported_as_a_product_defect():
+    result = resolve_case_result(
+        {
+            "tests": {
+                "failed": 1,
+                "failed_items": [
+                    {"kind": "verify", "item_text": "打开设置", "evidence": "页面未打开"}
+                ],
+            }
+        },
+        "failed",
+    )
+    assert result.status == "blocked"
+
+
+def test_legacy_completed_flash_without_verification_is_inconclusive():
+    assert resolve_case_result({}, "completed").status == "inconclusive"
+    assert resolve_case_result(None, "running") is None
+    assert resolve_case_result({"tests": {"passed": 2}}, "completed").status == "passed"
+
+
+def test_missing_checks_prevent_an_overall_pass():
+    assert (
+        resolve_case_result(outcome(tests={"passed": 1, "unchecked": 1}), "completed").status
+        == "inconclusive"
+    )
+
+
+def test_session_reader_handles_missing_and_corrupt_outcomes_without_claiming_pass(tmp_path):
+    assert read_case_result("case", "completed", root=tmp_path)["status"] == "inconclusive"
+    directory = tmp_path / "case"
+    directory.mkdir()
+    (directory / "run_outcome.json").write_text("{", encoding="utf-8")
+    assert read_case_result("case", "completed", root=tmp_path)["status"] == "inconclusive"
+    (directory / "run_outcome.json").write_text(json.dumps(outcome("failed")), encoding="utf-8")
+    assert read_case_result("case", "completed", root=tmp_path)["status"] == "failed"
+    with pytest.raises(ValueError):
+        read_case_result("../case", "completed", root=tmp_path)
+
+
+def test_failure_persistence_keeps_confirmed_test_failure_and_checkpoint_evidence(
+    tmp_path, monkeypatch
+):
+    from artemis.runtime import task_result
+
+    monkeypatch.setattr(task_result, "TRACES_PATH", tmp_path)
+    directory = tmp_path / "case"
+    directory.mkdir()
+    (directory / "result.json").write_text(
+        json.dumps(outcome("failed", tests={"checkpoint": {"passed": False}})),
+        encoding="utf-8",
+    )
+    (directory / "run_outcome.json").write_text(
+        json.dumps(outcome("passed", tests={"passed": 1})), encoding="utf-8"
+    )
+    task_result.save_failure_result("case", "任务超时")
+    for name in ("result.json", "run_outcome.json"):
+        saved = json.loads((directory / name).read_text(encoding="utf-8"))
+        assert saved["test_result"]["status"] == "failed"
+        assert saved["tests"]["checkpoint"]["passed"] is False
+        assert saved["test_result"]["evidence"] == ["步骤 2 截图显示实际结果"]