import json import pytest from artemis.runtime.case_result import ( CaseTestResult, normalize_case_result, read_case_result, resolve_case_result, ) from artemis.sdk.agent import attach_test_summary, resolve_trace_suffix def outcome(status="passed", **extra): return { "task_status": "completed", "test_result": { "status": status, "reason": "预期与实际对比", "evidence": ["步骤 2 截图显示实际结果"], }, **extra, } @pytest.mark.parametrize("status", ["passed", "failed"]) def test_a_claim_without_evidence_is_never_a_verdict(status): result = normalize_case_result( CaseTestResult(status=status, reason="执行者说完成了", evidence=[" "]) ) assert result.status == "inconclusive" assert "证据" in result.reason def test_completed_execution_and_failed_test_are_independent(): result = resolve_case_result(outcome("failed"), "completed") assert result.status == "failed" returned = attach_test_summary( {"status": "completed", "explanation": "已完成", "test_result": {"status": "passed"}}, outcome("failed"), ) assert returned["status"] == "completed" assert returned["test_result"]["status"] == "failed" assert resolve_trace_suffix("completed", outcome("failed")) == "_TESTFAIL" @pytest.mark.parametrize("status", ["failed", "cancelled", "interrupted", "skipped"]) def test_incomplete_execution_does_not_inherit_a_pass(status): result = resolve_case_result(outcome(), status, "环境初始化失败") assert result.status == "blocked" assert result.reason == "环境初始化失败" def test_a_previously_confirmed_assertion_failure_survives_interruption_and_later_pass(): data = outcome( tests={ "failed": 1, "failed_items": [ { "kind": "assert", "item_text": "预期显示成功提示", "evidence": "步骤 4 正确点击后显示错误提示", } ], } ) result = resolve_case_result(data, "interrupted", "设备掉线") assert result.status == "failed" assert result.evidence == ["步骤 4 正确点击后显示错误提示"] def test_navigation_failure_is_not_reported_as_a_product_defect(): result = resolve_case_result( { "tests": { "failed": 1, "failed_items": [ {"kind": "verify", "item_text": "打开设置", "evidence": "页面未打开"} ], } }, "failed", ) assert result.status == "blocked" def test_legacy_completed_flash_without_verification_is_inconclusive(): assert resolve_case_result({}, "completed").status == "inconclusive" assert resolve_case_result(None, "running") is None assert resolve_case_result({"tests": {"passed": 2}}, "completed").status == "passed" def test_missing_checks_prevent_an_overall_pass(): assert ( resolve_case_result(outcome(tests={"passed": 1, "unchecked": 1}), "completed").status == "inconclusive" ) def test_session_reader_handles_missing_and_corrupt_outcomes_without_claiming_pass(tmp_path): assert read_case_result("case", "completed", root=tmp_path)["status"] == "inconclusive" directory = tmp_path / "case" directory.mkdir() (directory / "run_outcome.json").write_text("{", encoding="utf-8") assert read_case_result("case", "completed", root=tmp_path)["status"] == "inconclusive" (directory / "run_outcome.json").write_text(json.dumps(outcome("failed")), encoding="utf-8") assert read_case_result("case", "completed", root=tmp_path)["status"] == "failed" with pytest.raises(ValueError): read_case_result("../case", "completed", root=tmp_path) def test_failure_persistence_keeps_confirmed_test_failure_and_checkpoint_evidence( tmp_path, monkeypatch ): from artemis.runtime import task_result monkeypatch.setattr(task_result, "TRACES_PATH", tmp_path) directory = tmp_path / "case" directory.mkdir() (directory / "result.json").write_text( json.dumps(outcome("failed", tests={"checkpoint": {"passed": False}})), encoding="utf-8", ) (directory / "run_outcome.json").write_text( json.dumps(outcome("passed", tests={"passed": 1})), encoding="utf-8" ) task_result.save_failure_result("case", "任务超时") for name in ("result.json", "run_outcome.json"): saved = json.loads((directory / name).read_text(encoding="utf-8")) assert saved["test_result"]["status"] == "failed" assert saved["tests"]["checkpoint"]["passed"] is False assert saved["test_result"]["evidence"] == ["步骤 2 截图显示实际结果"]