133 lines
4.7 KiB
Python
133 lines
4.7 KiB
Python
import json
|
|
|
|
import pytest
|
|
|
|
from artemis.runtime.case_result import (
|
|
CaseTestResult,
|
|
normalize_case_result,
|
|
read_case_result,
|
|
resolve_case_result,
|
|
)
|
|
from artemis.sdk.agent import attach_test_summary, resolve_trace_suffix
|
|
|
|
|
|
def outcome(status="passed", **extra):
|
|
return {
|
|
"task_status": "completed",
|
|
"test_result": {
|
|
"status": status,
|
|
"reason": "预期与实际对比",
|
|
"evidence": ["步骤 2 截图显示实际结果"],
|
|
},
|
|
**extra,
|
|
}
|
|
|
|
|
|
@pytest.mark.parametrize("status", ["passed", "failed"])
|
|
def test_a_claim_without_evidence_is_never_a_verdict(status):
|
|
result = normalize_case_result(
|
|
CaseTestResult(status=status, reason="执行者说完成了", evidence=[" "])
|
|
)
|
|
assert result.status == "inconclusive"
|
|
assert "证据" in result.reason
|
|
|
|
|
|
def test_completed_execution_and_failed_test_are_independent():
|
|
result = resolve_case_result(outcome("failed"), "completed")
|
|
assert result.status == "failed"
|
|
returned = attach_test_summary(
|
|
{"status": "completed", "explanation": "已完成", "test_result": {"status": "passed"}},
|
|
outcome("failed"),
|
|
)
|
|
assert returned["status"] == "completed"
|
|
assert returned["test_result"]["status"] == "failed"
|
|
assert resolve_trace_suffix("completed", outcome("failed")) == "_TESTFAIL"
|
|
|
|
|
|
@pytest.mark.parametrize("status", ["failed", "cancelled", "interrupted", "skipped"])
|
|
def test_incomplete_execution_does_not_inherit_a_pass(status):
|
|
result = resolve_case_result(outcome(), status, "环境初始化失败")
|
|
assert result.status == "blocked"
|
|
assert result.reason == "环境初始化失败"
|
|
|
|
|
|
def test_a_previously_confirmed_assertion_failure_survives_interruption_and_later_pass():
|
|
data = outcome(
|
|
tests={
|
|
"failed": 1,
|
|
"failed_items": [
|
|
{
|
|
"kind": "assert",
|
|
"item_text": "预期显示成功提示",
|
|
"evidence": "步骤 4 正确点击后显示错误提示",
|
|
}
|
|
],
|
|
}
|
|
)
|
|
result = resolve_case_result(data, "interrupted", "设备掉线")
|
|
assert result.status == "failed"
|
|
assert result.evidence == ["步骤 4 正确点击后显示错误提示"]
|
|
|
|
|
|
def test_navigation_failure_is_not_reported_as_a_product_defect():
|
|
result = resolve_case_result(
|
|
{
|
|
"tests": {
|
|
"failed": 1,
|
|
"failed_items": [
|
|
{"kind": "verify", "item_text": "打开设置", "evidence": "页面未打开"}
|
|
],
|
|
}
|
|
},
|
|
"failed",
|
|
)
|
|
assert result.status == "blocked"
|
|
|
|
|
|
def test_legacy_completed_flash_without_verification_is_inconclusive():
|
|
assert resolve_case_result({}, "completed").status == "inconclusive"
|
|
assert resolve_case_result(None, "running") is None
|
|
assert resolve_case_result({"tests": {"passed": 2}}, "completed").status == "passed"
|
|
|
|
|
|
def test_missing_checks_prevent_an_overall_pass():
|
|
assert (
|
|
resolve_case_result(outcome(tests={"passed": 1, "unchecked": 1}), "completed").status
|
|
== "inconclusive"
|
|
)
|
|
|
|
|
|
def test_session_reader_handles_missing_and_corrupt_outcomes_without_claiming_pass(tmp_path):
|
|
assert read_case_result("case", "completed", root=tmp_path)["status"] == "inconclusive"
|
|
directory = tmp_path / "case"
|
|
directory.mkdir()
|
|
(directory / "run_outcome.json").write_text("{", encoding="utf-8")
|
|
assert read_case_result("case", "completed", root=tmp_path)["status"] == "inconclusive"
|
|
(directory / "run_outcome.json").write_text(json.dumps(outcome("failed")), encoding="utf-8")
|
|
assert read_case_result("case", "completed", root=tmp_path)["status"] == "failed"
|
|
with pytest.raises(ValueError):
|
|
read_case_result("../case", "completed", root=tmp_path)
|
|
|
|
|
|
def test_failure_persistence_keeps_confirmed_test_failure_and_checkpoint_evidence(
|
|
tmp_path, monkeypatch
|
|
):
|
|
from artemis.runtime import task_result
|
|
|
|
monkeypatch.setattr(task_result, "TRACES_PATH", tmp_path)
|
|
directory = tmp_path / "case"
|
|
directory.mkdir()
|
|
(directory / "result.json").write_text(
|
|
json.dumps(outcome("failed", tests={"checkpoint": {"passed": False}})),
|
|
encoding="utf-8",
|
|
)
|
|
(directory / "run_outcome.json").write_text(
|
|
json.dumps(outcome("passed", tests={"passed": 1})), encoding="utf-8"
|
|
)
|
|
task_result.save_failure_result("case", "任务超时")
|
|
for name in ("result.json", "run_outcome.json"):
|
|
saved = json.loads((directory / name).read_text(encoding="utf-8"))
|
|
assert saved["test_result"]["status"] == "failed"
|
|
assert saved["tests"]["checkpoint"]["passed"] is False
|
|
assert saved["test_result"]["evidence"] == ["步骤 2 截图显示实际结果"]
|