feat: add shared test acceptance and release 1.5.0

tests/unit/admin_console/test_automation.py

@@ -780,7 +780,9 @@ def test_failed_report_exposes_logged_reason_and_formats_chinese_markdown(isolat
     )
     report = reports.session_report(job["session_id"])
     assert "连续 3 次" in report["error"]
-    assert report["verdict"] == "未验证（任务异常终止）"
+    assert report["verdict"] == "测试阻塞"
+    assert report["test_result"]["status"] == "blocked"
+    assert "连续 3 次" in report["test_result"]["reason"]
     assert "abc123secret" not in reports.failure_reason(
         folder, "failed", "认证失败 OPENAI_API_KEY=abc123secret Authorization: Bearer abc123secret"
     )
@@ -796,7 +798,10 @@ def test_failed_report_exposes_logged_reason_and_formats_chinese_markdown(isolat
         cell.text == "分类" for table in doc.tables for row in table.rows for cell in row.cells
     )
     wb = load_workbook(next(path for path in paths if path.suffix == ".xlsx"))
-    assert "连续 3 次" in wb["任务结果"]["B15"].value
+    assert any(
+        row[0].value == "异常原因" and "连续 3 次" in row[1].value
+        for row in wb["任务结果"].iter_rows()
+    )
     assert wb["结论"]["B2"].value == "结果"
     assert wb["结果表1"]["A2"].value == "系统"
     assert wb["执行证据"]["A1"].font.name == "微软雅黑"
@@ -810,6 +815,51 @@ def test_failed_report_exposes_logged_reason_and_formats_chinese_markdown(isolat
     ]
 
 
+def test_prompt_based_verdict_is_consistent_in_history_batch_and_reports(
+    client, isolated, monkeypatch
+):
+    from artemis.runtime import case_result
+
+    monkeypatch.setattr(case_result, "TRACES_PATH", reports.TRACES_PATH)
+    batch = service.create_batch(payload())
+    job = isolated.jobs(batch["id"])[0]
+    isolated.save_job(
+        {**job, "status": "completed", "started_at": time.time() - 50, "ended_at": time.time()}
+    )
+    result = {
+        "status": "failed",
+        "reason": "预期出现成功提示，实际显示错误",
+        "evidence": ["步骤 3 点击后截图显示错误提示"],
+    }
+    folder = reports.TRACES_PATH / job["session_id"]
+    folder.mkdir(parents=True)
+    (folder / "run_outcome.json").write_text(
+        json.dumps(
+            {"task_status": "completed", "tests": {}, "test_result": result}, ensure_ascii=False
+        ),
+        encoding="utf-8",
+    )
+    history = client.get("/api/automation/history").json()["items"][0]
+    batch_job = client.get(f"/api/automation/batches/{batch['id']}").json()["jobs"][0]
+    report = client.get(f"/api/automation/reports/{job['session_id']}").json()
+    for item in (history, batch_job, report):
+        assert item["status"] == "completed"
+        assert item["test_result"] == result
+    assert report["verdict"] == "测试不通过"
+    paths = reports.export_session(job["session_id"])
+    doc = Document(next(path for path in paths if path.suffix == ".docx"))
+    doc_text = "\n".join(
+        cell.text for table in doc.tables for row in table.rows for cell in row.cells
+    )
+    doc_text += "\n" + "\n".join(paragraph.text for paragraph in doc.paragraphs)
+    assert result["reason"] in doc_text
+    assert result["evidence"][0] in doc_text
+    wb = load_workbook(next(path for path in paths if path.suffix == ".xlsx"))
+    values = [cell.value for row in wb["任务结果"].iter_rows() for cell in row]
+    assert "测试不通过" in values
+    assert result["reason"] in values
+
+
 def test_task_material_zip_contains_ordered_evidence_and_reports(
     client, isolated, tmp_path, monkeypatch
 ):

tests/unit/agents/test_flash_acceptance.py

@@ -0,0 +1,118 @@
+import asyncio
+import json
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from artemis.agents.checker import flash
+from artemis.agents.checker.checker import CheckReport
+from artemis.context import ArtemisContext, ExecutionSetup
+from artemis.runtime.case_result import CaseTestResult
+
+
+def context(tmp_path, **setup):
+    ctx = MagicMock(spec=ArtemisContext)
+    ctx.execution_setup = ExecutionSetup(**setup)
+    ctx.data_engine = MagicMock()
+    ctx.data_engine.base_dir = tmp_path
+    return ctx
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("verdict", ["passed", "failed", "inconclusive"])
+async def test_flash_reuses_checker_with_original_prompt_and_final_report(
+    tmp_path, monkeypatch, verdict
+):
+    result = CaseTestResult(
+        status=verdict, reason="提示词预期与实际对比", evidence=["步骤 3 截图显示实际状态"]
+    )
+    check = AsyncMock(return_value=CheckReport(test_result=result))
+    monkeypatch.setattr(flash, "run_final_check", check)
+    ctx = context(tmp_path)
+    execution = {"status": "completed", "explanation": "操作已执行"}
+    data = await flash.settle_flash_outcome(ctx, "打开设置，预期看到应用入口", execution)
+    assert check.call_count == 1
+    assert check.call_args.kwargs["goal"] == "打开设置，预期看到应用入口"
+    assert check.call_args.kwargs["execution_report"] == execution
+    assert check.call_args.kwargs["check_items"] == []
+    assert data["task_status"] == "completed"
+    assert data["test_result"]["status"] == verdict
+    assert (
+        json.loads((tmp_path / "run_outcome.json").read_text(encoding="utf-8"))["test_result"]
+        == data["test_result"]
+    )
+    assert ctx.run_outcome == data
+
+
+@pytest.mark.asyncio
+async def test_acceptance_timeout_finishes_with_unknown_verdict(tmp_path, monkeypatch):
+    async def hanging_check(*args, **kwargs):
+        await asyncio.Event().wait()
+
+    monkeypatch.setattr(flash, "run_final_check", hanging_check)
+    data = await flash.settle_flash_outcome(
+        context(tmp_path, checkpoint_timeout=0.01), "任务", {"status": "completed"}
+    )
+    assert data["task_status"] == "completed"
+    assert data["test_result"]["status"] == "inconclusive"
+    assert "0.01 秒" in data["test_result"]["reason"]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("setup", [{"disable_checker": True}, {"disable_final_check": True}])
+async def test_disabled_acceptance_does_not_call_a_model_or_claim_pass(
+    tmp_path, monkeypatch, setup
+):
+    check = AsyncMock()
+    monkeypatch.setattr(flash, "run_final_check", check)
+    data = await flash.settle_flash_outcome(
+        context(tmp_path, **setup), "任务", {"status": "completed"}
+    )
+    check.assert_not_called()
+    assert data["test_result"]["status"] == "inconclusive"
+
+
+@pytest.mark.asyncio
+async def test_execution_failure_is_blocked_without_another_model_retry(tmp_path, monkeypatch):
+    check = AsyncMock()
+    monkeypatch.setattr(flash, "run_final_check", check)
+    data = await flash.settle_flash_outcome(
+        context(tmp_path), "任务", {"status": "failed", "explanation": "设备观测失败"}
+    )
+    check.assert_not_called()
+    assert data["test_result"]["status"] == "blocked"
+    assert data["test_result"]["reason"] == "设备观测失败"
+
+
+@pytest.mark.asyncio
+async def test_reported_expectation_mismatch_is_checked_even_when_executor_reports_failed(
+    tmp_path, monkeypatch
+):
+    check = AsyncMock(
+        return_value=CheckReport(
+            test_result=CaseTestResult(
+                status="failed",
+                reason="预期英文 Search settings，实际中文 搜索设置。",
+                evidence=["设置首页截图和 UI 元素 1 显示 搜索设置"],
+            )
+        )
+    )
+    monkeypatch.setattr(flash, "run_final_check", check)
+    data = await flash.settle_flash_outcome(
+        context(tmp_path),
+        "预期搜索框为英文 Search settings",
+        {"status": "failed", "explanation": "已检查，实际为中文"},
+        final_report_received=True,
+    )
+    check.assert_awaited_once()
+    assert data["test_result"]["status"] == "failed"
+    assert "Search settings" in data["test_result"]["reason"]
+
+
+@pytest.mark.asyncio
+async def test_cancellation_of_acceptance_propagates(tmp_path, monkeypatch):
+    check = AsyncMock(side_effect=asyncio.CancelledError)
+    monkeypatch.setattr(flash, "run_final_check", check)
+    with pytest.raises(asyncio.CancelledError):
+        await flash.settle_flash_outcome(context(tmp_path), "任务", {"status": "completed"})
+    assert not (tmp_path / "run_outcome.json").exists()

tests/unit/test_case_result.py

@@ -0,0 +1,132 @@
+import json
+
+import pytest
+
+from artemis.runtime.case_result import (
+    CaseTestResult,
+    normalize_case_result,
+    read_case_result,
+    resolve_case_result,
+)
+from artemis.sdk.agent import attach_test_summary, resolve_trace_suffix
+
+
+def outcome(status="passed", **extra):
+    return {
+        "task_status": "completed",
+        "test_result": {
+            "status": status,
+            "reason": "预期与实际对比",
+            "evidence": ["步骤 2 截图显示实际结果"],
+        },
+        **extra,
+    }
+
+
+@pytest.mark.parametrize("status", ["passed", "failed"])
+def test_a_claim_without_evidence_is_never_a_verdict(status):
+    result = normalize_case_result(
+        CaseTestResult(status=status, reason="执行者说完成了", evidence=[" "])
+    )
+    assert result.status == "inconclusive"
+    assert "证据" in result.reason
+
+
+def test_completed_execution_and_failed_test_are_independent():
+    result = resolve_case_result(outcome("failed"), "completed")
+    assert result.status == "failed"
+    returned = attach_test_summary(
+        {"status": "completed", "explanation": "已完成", "test_result": {"status": "passed"}},
+        outcome("failed"),
+    )
+    assert returned["status"] == "completed"
+    assert returned["test_result"]["status"] == "failed"
+    assert resolve_trace_suffix("completed", outcome("failed")) == "_TESTFAIL"
+
+
+@pytest.mark.parametrize("status", ["failed", "cancelled", "interrupted", "skipped"])
+def test_incomplete_execution_does_not_inherit_a_pass(status):
+    result = resolve_case_result(outcome(), status, "环境初始化失败")
+    assert result.status == "blocked"
+    assert result.reason == "环境初始化失败"
+
+
+def test_a_previously_confirmed_assertion_failure_survives_interruption_and_later_pass():
+    data = outcome(
+        tests={
+            "failed": 1,
+            "failed_items": [
+                {
+                    "kind": "assert",
+                    "item_text": "预期显示成功提示",
+                    "evidence": "步骤 4 正确点击后显示错误提示",
+                }
+            ],
+        }
+    )
+    result = resolve_case_result(data, "interrupted", "设备掉线")
+    assert result.status == "failed"
+    assert result.evidence == ["步骤 4 正确点击后显示错误提示"]
+
+
+def test_navigation_failure_is_not_reported_as_a_product_defect():
+    result = resolve_case_result(
+        {
+            "tests": {
+                "failed": 1,
+                "failed_items": [
+                    {"kind": "verify", "item_text": "打开设置", "evidence": "页面未打开"}
+                ],
+            }
+        },
+        "failed",
+    )
+    assert result.status == "blocked"
+
+
+def test_legacy_completed_flash_without_verification_is_inconclusive():
+    assert resolve_case_result({}, "completed").status == "inconclusive"
+    assert resolve_case_result(None, "running") is None
+    assert resolve_case_result({"tests": {"passed": 2}}, "completed").status == "passed"
+
+
+def test_missing_checks_prevent_an_overall_pass():
+    assert (
+        resolve_case_result(outcome(tests={"passed": 1, "unchecked": 1}), "completed").status
+        == "inconclusive"
+    )
+
+
+def test_session_reader_handles_missing_and_corrupt_outcomes_without_claiming_pass(tmp_path):
+    assert read_case_result("case", "completed", root=tmp_path)["status"] == "inconclusive"
+    directory = tmp_path / "case"
+    directory.mkdir()
+    (directory / "run_outcome.json").write_text("{", encoding="utf-8")
+    assert read_case_result("case", "completed", root=tmp_path)["status"] == "inconclusive"
+    (directory / "run_outcome.json").write_text(json.dumps(outcome("failed")), encoding="utf-8")
+    assert read_case_result("case", "completed", root=tmp_path)["status"] == "failed"
+    with pytest.raises(ValueError):
+        read_case_result("../case", "completed", root=tmp_path)
+
+
+def test_failure_persistence_keeps_confirmed_test_failure_and_checkpoint_evidence(
+    tmp_path, monkeypatch
+):
+    from artemis.runtime import task_result
+
+    monkeypatch.setattr(task_result, "TRACES_PATH", tmp_path)
+    directory = tmp_path / "case"
+    directory.mkdir()
+    (directory / "result.json").write_text(
+        json.dumps(outcome("failed", tests={"checkpoint": {"passed": False}})),
+        encoding="utf-8",
+    )
+    (directory / "run_outcome.json").write_text(
+        json.dumps(outcome("passed", tests={"passed": 1})), encoding="utf-8"
+    )
+    task_result.save_failure_result("case", "任务超时")
+    for name in ("result.json", "run_outcome.json"):
+        saved = json.loads((directory / name).read_text(encoding="utf-8"))
+        assert saved["test_result"]["status"] == "failed"
+        assert saved["tests"]["checkpoint"]["passed"] is False
+        assert saved["test_result"]["evidence"] == ["步骤 2 截图显示实际结果"]