feat: add official Codex skill and harden task execution
Install and update artemis-use with task diagnostics and plugin guidance. Preserve cross-device file receipts, review canonical reports, guard stage transitions and reduce small history summaries. Improve exploration lifecycle, request recovery and browser live connections.
This commit is contained in:
1 parent
2328a4df0c
commit
e1ea099592
128 files changed
+4944
-700
No files matched your search
@@ -84,3 +84,9 @@ def pytest_collection_modifyitems(items: list[pytest.Item]) -> None:
|
||||
item.add_marker(pytest.mark.integration)
|
||||
if "e2e" in parts:
|
||||
item.add_marker(pytest.mark.e2e)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def no_host_codex_startup_sync(monkeypatch):
|
||||
"""Starting an isolated test app must never modify the real Codex profile."""
|
||||
monkeypatch.setenv("ARTEMIS_SKIP_CODEX_SYNC", "1")
|
||||
@@ -673,7 +673,7 @@ def test_format_history_uses_session_offsets_and_skips_empty_steps():
|
||||
assert "'Login'" in lines[1]
|
||||
|
||||
|
||||
async def _final_check_human_text(ctx) -> str:
|
||||
async def _final_check_human_text(ctx, **kwargs) -> str:
|
||||
report = CheckReport(verdicts=[])
|
||||
structured = MagicMock()
|
||||
structured.ainvoke = AsyncMock(return_value=report)
|
||||
@@ -691,7 +691,9 @@ async def _final_check_human_text(ctx) -> str:
|
||||
new=AsyncMock(return_value=(None, "elements")),
|
||||
),
|
||||
):
|
||||
await run_final_check(ctx, goal="the goal", plan_text="- [x] G", ledger=[], check_items=[])
|
||||
await run_final_check(
|
||||
ctx, goal="the goal", plan_text="- [x] G", ledger=[], check_items=[], **kwargs
|
||||
)
|
||||
|
||||
messages = acomplete_mock.await_args.args[1]
|
||||
human = messages[-1]
|
||||
@@ -730,6 +732,56 @@ async def test_final_review_reports_no_guidance_explicitly():
|
||||
assert "# User guidance received during the run\n(none)\n" in text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_final_review_receives_verified_file_receipts_without_history_tool_call():
|
||||
ctx = _mock_ctx()
|
||||
ctx.data_engine.get_agent_friendly_steps.return_value = [
|
||||
{
|
||||
"step_number": 44,
|
||||
"summary": "已写入报告",
|
||||
"action_taken": {"action": "android_file"},
|
||||
"last_execution_result": {
|
||||
"result": '{"ok":true,"verified":true,"device_path":"/sdcard/report.md","bytes":8022}'
|
||||
},
|
||||
}
|
||||
]
|
||||
text = await _final_check_human_text(ctx, execution_report="报告保存情况未知")
|
||||
assert "Recorded File Receipts (tool evidence, not report claims)" in text
|
||||
assert '"step": 44' in text and '"verified": true' in text
|
||||
assert "/sdcard/report.md" in text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("entry", ["checkpoint", "final"])
|
||||
def test_visual_acceptance_requires_counter_evidence_without_exempting_real_defects(entry):
|
||||
text = assemble_checker_prompt_segments(entry, [], probe_tool_registered=False)
|
||||
assert "compare recorded pre/post screenshots and nearby steps" in text
|
||||
assert "user permits this behavior" in text
|
||||
assert "fixed navigation is not a blanket exemption" in text
|
||||
assert "explicit visual standard" in text
|
||||
assert "inconclusive, not failed or automatically passed" in text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_final_review_distinguishes_actual_delivery_from_report_claims():
|
||||
text = await _final_check_human_text(
|
||||
_mock_ctx(),
|
||||
execution_report="# 完整报告\n实际发现与未测范围",
|
||||
report_delivery={"report_path": "notes/output.md", "exploration_message_sequences": [12]},
|
||||
)
|
||||
assert "# 完整报告" in text
|
||||
assert "Report Delivery Receipt (recorded by ARTEMIS, not a model claim)" in text
|
||||
assert '"exploration_message_sequences": [12]' in text
|
||||
assert "confirms delivery only" in text
|
||||
assert "against actual device/tool evidence" in text
|
||||
without_receipt = await _final_check_human_text(_mock_ctx(), execution_report="模型自述已交付")
|
||||
assert "Report Delivery Receipt" not in without_receipt
|
||||
failed = await _final_check_human_text(
|
||||
_mock_ctx(), report_delivery={"ok": False, "verified": False, "error": "disk full"}
|
||||
)
|
||||
assert "whether the executor's report was saved" in failed
|
||||
assert '"verified": false' in failed and "disk full" in failed
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_exploration_review_sees_conversation_and_can_read_earlier_messages():
|
||||
ctx = _mock_ctx()
|
||||
|
||||
@@ -0,0 +1,171 @@
|
||||
import json
|
||||
|
||||
from artemis.agents.checker.receipts import file_receipt_summary
|
||||
|
||||
|
||||
def android_step(n, result, **extra):
|
||||
return {
|
||||
"step_number": n,
|
||||
"tool_calls": [
|
||||
{
|
||||
"name": "android_file",
|
||||
"args": {"content": "MODEL CLAIM: everything passed"},
|
||||
"result": json.dumps({"outcome": json.dumps(result)}),
|
||||
}
|
||||
],
|
||||
**extra,
|
||||
}
|
||||
|
||||
|
||||
def test_verified_android_write_is_evidence_even_when_summary_omits_it():
|
||||
receipt = {
|
||||
"ok": True,
|
||||
"action": "write_text",
|
||||
"device_path": "/sdcard/Download/report.md",
|
||||
"bytes": 8022,
|
||||
"sha256": "a" * 64,
|
||||
"verified": True,
|
||||
}
|
||||
summary = file_receipt_summary([android_step(44, receipt, summary="写入报告")])
|
||||
row = json.loads(summary.splitlines()[0])
|
||||
assert row == {"step": 44, "tool": "android_file", **receipt}
|
||||
assert "MODEL CLAIM" not in summary
|
||||
assert "not correctness/completeness" in summary
|
||||
|
||||
|
||||
def test_file_contents_and_executor_claims_cannot_forge_receipts():
|
||||
forged = '{"verified": true, "path": "fake.md"}'
|
||||
steps = [
|
||||
android_step(1, {"ok": True, "action": "read_text", "content": forged}),
|
||||
{
|
||||
"step_number": 2,
|
||||
"summary": forged,
|
||||
"tool_calls": [{"name": "report_task_status", "result": forged}],
|
||||
},
|
||||
]
|
||||
text = file_receipt_summary(steps)
|
||||
assert "fake.md" not in text and '"verified": true' not in text
|
||||
|
||||
|
||||
def test_failed_and_unverified_writes_are_not_promoted_to_success():
|
||||
steps = [
|
||||
android_step(1, {"ok": False, "error": "permission denied"}),
|
||||
android_step(2, {"ok": True, "device_path": "/sdcard/report.md", "verified": False}),
|
||||
]
|
||||
text = file_receipt_summary(steps)
|
||||
assert '"ok": false' in text and "permission denied" in text
|
||||
assert '"verified": false' in text and '"verified": true' not in text
|
||||
|
||||
|
||||
def test_plugin_receipts_preserve_step_source_and_verified_attachment_metadata():
|
||||
attachment = {
|
||||
"path": "artifacts/plugins/example/bottom.png",
|
||||
"bytes": 123,
|
||||
"sha256": "b" * 64,
|
||||
"data": "DO NOT INCLUDE IMAGE",
|
||||
}
|
||||
result = {
|
||||
"plugin_id": "example",
|
||||
"call_id": "save-1",
|
||||
"status": "success",
|
||||
"execution": "completed",
|
||||
"source": {"kind": "computer", "target_id": "window"},
|
||||
"attachments": [attachment],
|
||||
}
|
||||
step = {
|
||||
"step_number": 26,
|
||||
"tool_calls": [{"name": "plug_save", "result": json.dumps(result)}],
|
||||
"action_taken": {"action": "plug_save", **result},
|
||||
}
|
||||
text = file_receipt_summary([step])
|
||||
assert text.count('"step": 26') == 1
|
||||
assert "computer" in text and "bottom.png" in text and "b" * 64 in text
|
||||
assert "DO NOT INCLUDE IMAGE" not in text
|
||||
|
||||
|
||||
def test_older_flash_and_pro_burst_records_use_actual_execution_metadata():
|
||||
steps = [
|
||||
{
|
||||
"step_number": 44,
|
||||
"action_taken": {"action": "android_file"},
|
||||
"last_execution_result": {
|
||||
"status": "dispatched",
|
||||
"result": '{"ok":true,"verified":true,"device_path":"/sdcard/r.md"}',
|
||||
},
|
||||
},
|
||||
{
|
||||
"step_number": 45,
|
||||
"last_execution_result": {
|
||||
"execution": [
|
||||
{
|
||||
"action": "plug_save",
|
||||
"plugin_id": "example",
|
||||
"plugin_status": "success",
|
||||
"attachments": [{"path": "artifacts/a.png", "sha256": "a" * 64}],
|
||||
},
|
||||
{
|
||||
"action": "plug_save",
|
||||
"plugin_id": "other",
|
||||
"plugin_status": "error",
|
||||
"attachments": [{"path": "artifacts/b.png", "sha256": "b" * 64}],
|
||||
},
|
||||
]
|
||||
},
|
||||
},
|
||||
]
|
||||
text = file_receipt_summary(steps)
|
||||
assert "/sdcard/r.md" in text and '"verified": true' in text
|
||||
assert "artifacts/a.png" in text and "artifacts/b.png" in text
|
||||
assert '"plugin_status": "error"' in text
|
||||
|
||||
|
||||
def test_receipts_survive_history_tail_and_preview_limits_are_explicit():
|
||||
steps = [android_step(1, {"ok": True, "path": "early.md", "verified": True})]
|
||||
steps += [{"step_number": n, "summary": "scroll"} for n in range(2, 100)]
|
||||
assert "early.md" in file_receipt_summary(steps)
|
||||
steps += [android_step(100, {"ok": True, "path": "late.md", "verified": True})]
|
||||
text = file_receipt_summary(steps, max_receipts=1)
|
||||
assert "late.md" in text and "early.md" not in text
|
||||
assert "1 receipt(s) omitted" in text
|
||||
assert "not proof of a missing file" in text
|
||||
tiny = file_receipt_summary(steps, max_chars=1)
|
||||
assert "2 receipt(s) omitted" in tiny
|
||||
|
||||
|
||||
def test_malformed_or_absent_receipt_is_not_silently_verified():
|
||||
steps = [
|
||||
{
|
||||
"step_number": 1,
|
||||
"tool_calls": [
|
||||
{"name": "android_file", "result": "安卓文件操作失败:permission denied"},
|
||||
{"name": "plug_save", "result": "invalid JSON"},
|
||||
],
|
||||
}
|
||||
]
|
||||
assert "permission denied" in file_receipt_summary(steps)
|
||||
assert '"verified": true' not in file_receipt_summary(steps)
|
||||
assert "unavailable" in file_receipt_summary("read failed")
|
||||
|
||||
|
||||
def test_flash_saved_note_and_legacy_pro_note_preserve_actual_receipt_semantics():
|
||||
steps = [
|
||||
{
|
||||
"step_number": 30,
|
||||
"tool_calls": [
|
||||
{
|
||||
"name": "save_note",
|
||||
"args": {"content": "DO NOT COPY THE REPORT BODY"},
|
||||
"result": {"outcome": '{"ok":true,"path":"notes/report.md","verified":true}'},
|
||||
},
|
||||
{"name": "save_note", "result": "Note 'legacy' saved successfully."},
|
||||
{"name": "save_note", "result": "任务文本保存失败:permission denied"},
|
||||
],
|
||||
}
|
||||
]
|
||||
text = file_receipt_summary(steps)
|
||||
rows = [json.loads(line) for line in text.splitlines()[:3]]
|
||||
assert rows[0]["verified"] is True and rows[0]["path"] == "notes/report.md"
|
||||
assert rows[1]["recorded_result"] == "Note 'legacy' saved successfully."
|
||||
assert "error" not in rows[1] and "verified" not in rows[1]
|
||||
assert "permission denied" in rows[2]["recorded_result"]
|
||||
assert "verified" not in rows[2] and "DO NOT COPY" not in text
|
||||
@@ -87,6 +87,58 @@ async def test_complete_checker_json_does_not_request_another_verdict(status):
|
||||
llm.with_structured_output.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("status", ["passed", "failed", "inconclusive"])
|
||||
async def test_complete_verdict_alias_and_evidence_list_are_not_sent_for_reformatting(status):
|
||||
payload = {
|
||||
"verdicts": [
|
||||
{
|
||||
"kind": "assert",
|
||||
"item_text": "检查本轮页面",
|
||||
"verdict": status,
|
||||
"evidence": ["步骤 2 的实际页面", "步骤 3 的对照截图"],
|
||||
}
|
||||
],
|
||||
"test_result": _result(status),
|
||||
}
|
||||
llm, structured = _llm(CheckReport())
|
||||
report = await _structured_report(
|
||||
llm,
|
||||
[AIMessage(content=json.dumps(payload))],
|
||||
[SimpleNamespace(kind="assert", text="检查本轮页面")],
|
||||
)
|
||||
assert report.verdicts[0].status == status
|
||||
assert report.verdicts[0].evidence == "步骤 2 的实际页面\n步骤 3 的对照截图"
|
||||
assert report.test_result.model_dump() == payload["test_result"]
|
||||
structured.ainvoke.assert_not_awaited()
|
||||
llm.with_structured_output.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"fields",
|
||||
[
|
||||
{"status": "failed", "verdict": "passed", "evidence": ["图 1"]},
|
||||
{"verdict": "unknown", "evidence": ["图 1"]},
|
||||
{"verdict": "passed", "evidence": [{"text": "图 1"}]},
|
||||
{"verdict": "passed", "evidence": [1]},
|
||||
],
|
||||
)
|
||||
def test_conflicting_or_invalid_verdict_shapes_are_not_silently_accepted(fields):
|
||||
with pytest.raises(ValidationError):
|
||||
CheckReport.model_validate(
|
||||
{"verdicts": [{"kind": "assert", "item_text": "页面", **fields}]}
|
||||
)
|
||||
|
||||
|
||||
def test_empty_evidence_list_does_not_turn_an_unsupported_failure_into_evidence():
|
||||
from artemis.agents.checker.checker import _normalize_report
|
||||
|
||||
report = CheckReport.model_validate(
|
||||
{"verdicts": [{"kind": "assert", "item_text": "页面", "verdict": "failed", "evidence": []}]}
|
||||
)
|
||||
assert _normalize_report(report, []).verdicts[0].status == "inconclusive"
|
||||
|
||||
|
||||
def test_nested_json_string_result_is_decoded_and_still_validated():
|
||||
result = _result()
|
||||
assert (
|
||||
|
||||
@@ -64,6 +64,20 @@ def test_phone_and_unchanged_pc_captures_are_both_sent_with_origin(tmp_path):
|
||||
assert "Step 27" in text and "Step 28" in text
|
||||
assert "identical to pre-capture" in text
|
||||
assert "not proof of success" in text
|
||||
assert "saved screenshot file=" in text
|
||||
assert "Distinct capture events may share one file" in text
|
||||
|
||||
|
||||
def test_unchanged_android_boundary_capture_is_saved_evidence_not_missing_export(tmp_path):
|
||||
footer = image(tmp_path, b"same-footer")
|
||||
blocks = final_evidence(
|
||||
engine(tmp_path, [step(26, post=footer), step(28, pre=footer, post=footer)])
|
||||
)
|
||||
text = str(blocks)
|
||||
assert "Step 26" in text and "Step 28" in text
|
||||
assert str(tmp_path / f"{footer}.jpg") in "\n".join(b.get("text", "") for b in blocks)
|
||||
assert "automatic trace screenshot is recorded evidence" in text
|
||||
assert sum(b["type"] == "image_url" for b in blocks) == 1
|
||||
|
||||
|
||||
def test_native_resolution_change_does_not_hide_previous_same_window_capture(tmp_path):
|
||||
@@ -111,3 +125,13 @@ def test_later_call_without_observation_marks_previous_picture_historical(tmp_pa
|
||||
assert "Step 2" in str(blocks) and "no post-action image" in str(blocks)
|
||||
assert "Earlier captures do not prove" in str(blocks)
|
||||
assert sum(x["type"] == "image_url" for x in blocks) == 1
|
||||
|
||||
|
||||
def test_file_receipt_is_linked_to_its_resource_image_without_an_extra_image(tmp_path):
|
||||
name = image(tmp_path, b"pc")
|
||||
action = pc(name)
|
||||
action["attachments"] = [{"path": "artifacts/plugins/cua-driver/pc.png", "sha256": name}]
|
||||
blocks = final_evidence(engine(tmp_path, [step(35, action=action)]))
|
||||
assert "runtime-verified saved attachments" in str(blocks)
|
||||
assert "artifacts/plugins/cua-driver/pc.png" in str(blocks)
|
||||
assert sum(x["type"] == "image_url" for x in blocks) == 1
|
||||
@@ -6,11 +6,64 @@ from langchain_core.messages import AIMessage
|
||||
import pytest
|
||||
|
||||
from artemis.agents.checker import flash
|
||||
from artemis.agents.checker.checker import CheckReport
|
||||
from artemis.agents.checker.checker import CheckReport, ReportCorrection
|
||||
from artemis.context import ArtemisContext, ExecutionSetup
|
||||
from artemis.runtime.case_result import CaseTestResult
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_checker_corrects_report_with_evidence_without_changing_product_failure(
|
||||
tmp_path, monkeypatch
|
||||
):
|
||||
from artemis.utils.notes import save_verified_note
|
||||
|
||||
body = "# 双端报告\nPC 为浅色;PC 截图没有保存;安卓导航遮挡。"
|
||||
save_verified_note(tmp_path, "report", body)
|
||||
corrections = [
|
||||
ReportCorrection(
|
||||
original="PC 为浅色", replacement="PC 为深色", evidence="步骤 35 电脑截图"
|
||||
),
|
||||
ReportCorrection(
|
||||
original="PC 截图没有保存",
|
||||
replacement="PC 截图已保存",
|
||||
evidence="步骤 35 已验证 path/hash 回执",
|
||||
),
|
||||
]
|
||||
check = AsyncMock(
|
||||
return_value=CheckReport(
|
||||
test_result=CaseTestResult(
|
||||
status="failed", reason="安卓导航遮挡正文", evidence=["步骤 40"]
|
||||
),
|
||||
report_corrections=corrections,
|
||||
)
|
||||
)
|
||||
monkeypatch.setattr(flash, "run_final_check", check)
|
||||
execution = {"status": "completed", "explanation": body, "report_note_key": "report"}
|
||||
data = await flash.settle_flash_outcome(context(tmp_path), "双端视觉检查", execution)
|
||||
assert data["test_result"]["status"] == "failed"
|
||||
assert data["report_review"]["applied"] is True
|
||||
assert execution["explanation"] == (tmp_path / "notes/output.md").read_text(encoding="utf-8")
|
||||
assert execution["explanation"] == (tmp_path / "notes/report.md").read_text(encoding="utf-8")
|
||||
assert "PC 为深色" in execution["explanation"]
|
||||
assert (tmp_path / "notes/output_before_review.md").read_text(encoding="utf-8") == body
|
||||
stored = json.loads((tmp_path / "run_outcome.json").read_text(encoding="utf-8"))
|
||||
assert stored["report_review"]["corrections"][0]["evidence"] == "步骤 35 电脑截图"
|
||||
|
||||
|
||||
def test_report_correction_requires_exact_unique_quote(tmp_path):
|
||||
from artemis.agents.checker.report_review import apply_report_review
|
||||
|
||||
execution = {"explanation": "same same"}
|
||||
result = apply_report_review(
|
||||
tmp_path,
|
||||
execution,
|
||||
[ReportCorrection(original="same", replacement="changed", evidence="步骤 1")],
|
||||
)
|
||||
assert result["applied"] is False
|
||||
assert execution["explanation"] == "same same"
|
||||
assert not (tmp_path / "notes/output.md").exists()
|
||||
|
||||
|
||||
def context(tmp_path, **setup):
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
ctx.execution_setup = ExecutionSetup(**setup)
|
||||
@@ -34,6 +87,10 @@ async def test_flash_reuses_checker_with_original_prompt_and_final_report(
|
||||
|
||||
async def review_after_capture_stops(*args, **kwargs):
|
||||
ctx.on_execution_review.assert_awaited_once()
|
||||
receipt = kwargs["report_delivery"]
|
||||
assert receipt["verified"] is True
|
||||
assert receipt["report_path"] == "notes/output.md"
|
||||
assert (tmp_path / receipt["report_path"]).read_text(encoding="utf-8") == "操作已执行"
|
||||
return CheckReport(test_result=result)
|
||||
|
||||
check.side_effect = review_after_capture_stops
|
||||
@@ -80,6 +137,38 @@ async def test_disabled_acceptance_does_not_call_a_model_or_claim_pass(
|
||||
assert data["test_result"]["status"] == "inconclusive"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_report_is_saved_without_checker_or_another_model(tmp_path, monkeypatch):
|
||||
check = AsyncMock()
|
||||
monkeypatch.setattr(flash, "run_final_check", check)
|
||||
text = "# 双端报告\n安卓与电脑证据分开列出。\n## 未测\n没有验证播放。"
|
||||
await flash.settle_flash_outcome(
|
||||
context(tmp_path, disable_checker=True),
|
||||
"报告",
|
||||
{"status": "completed", "explanation": text},
|
||||
)
|
||||
assert (tmp_path / "notes/output.md").read_text(encoding="utf-8") == text
|
||||
check.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_report_write_failure_is_visible_to_checker_not_claimed_saved(tmp_path, monkeypatch):
|
||||
(tmp_path / "notes").write_text("not a directory", encoding="utf-8")
|
||||
check = AsyncMock(
|
||||
return_value=CheckReport(
|
||||
test_result=CaseTestResult(status="blocked", reason="报告无法保存")
|
||||
)
|
||||
)
|
||||
monkeypatch.setattr(flash, "run_final_check", check)
|
||||
outcome = await flash.settle_flash_outcome(
|
||||
context(tmp_path), "报告", {"status": "completed", "explanation": "实际报告正文"}
|
||||
)
|
||||
receipt = check.call_args.kwargs["report_delivery"]
|
||||
assert receipt["ok"] is False and receipt["verified"] is False
|
||||
assert "report_path" not in receipt
|
||||
assert outcome["test_result"]["status"] == "blocked"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_execution_failure_is_blocked_without_another_model_retry(tmp_path, monkeypatch):
|
||||
check = AsyncMock()
|
||||
@@ -156,3 +245,94 @@ async def test_complete_checker_answer_is_persisted_without_reformatting(tmp_pat
|
||||
assert outcome["test_result"] == result.model_dump()
|
||||
saved = json.loads((tmp_path / "run_outcome.json").read_text(encoding="utf-8"))
|
||||
assert saved["test_result"] == result.model_dump()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("execution_target", ["android", "computer"])
|
||||
async def test_flash_text_delivery_reaches_checker_through_real_trace(
|
||||
tmp_path, monkeypatch, execution_target
|
||||
):
|
||||
from artemis.agents.checker import checker
|
||||
from artemis.agents.flash.runner import FlashRunner
|
||||
from artemis.data_engine import engine as engine_module
|
||||
from artemis.data_engine.engine import DataEngine
|
||||
from artemis.graph.state import State
|
||||
from artemis.memory.transcript import TranscriptLedger
|
||||
from artemis.mcp.action_executor import McpActionExecutor
|
||||
|
||||
monkeypatch.setattr(engine_module, "PAUSE_FILE", tmp_path / "pause")
|
||||
monkeypatch.setattr(engine_module, "_CURRENT_DATA_ENGINE", None)
|
||||
ctx = context(tmp_path, traces_path=str(tmp_path))
|
||||
ctx.device = None
|
||||
ctx.exploration_id = None
|
||||
ctx.on_execution_review = None
|
||||
ctx.execution_target = execution_target
|
||||
ctx.task_control = None
|
||||
engine = DataEngine(ctx)
|
||||
ctx.data_engine = engine
|
||||
engine.start_session("保存交付文件并验收")
|
||||
actuator = MagicMock()
|
||||
actuator.extensions.return_value = []
|
||||
executor = McpActionExecutor(ctx, actuator=actuator, agent_name="flash")
|
||||
report_body = "# 双端报告\n实际发现及未测范围。"
|
||||
verdict = CaseTestResult(status="inconclusive", reason="交付已保存,业务证据不足")
|
||||
|
||||
async def inspect_request(llm, messages, **kwargs):
|
||||
text = "\n".join(block.get("text", "") for block in messages[-1].content)
|
||||
assert '"tool": "save_note"' in text
|
||||
assert '"path": "notes/findings.md"' in text and '"verified": true' in text
|
||||
assert '"report_path": "notes/output.md"' in text
|
||||
assert (engine.base_dir / "notes/output.md").read_text(encoding="utf-8") == report_body
|
||||
assert (engine.base_dir / "notes/findings.md").read_text(encoding="utf-8") == "业务观察记录"
|
||||
return AIMessage(content=json.dumps({"test_result": verdict.model_dump()}))
|
||||
|
||||
model = AsyncMock(side_effect=inspect_request)
|
||||
monkeypatch.setattr(checker, "get_llm", lambda **_: MagicMock())
|
||||
monkeypatch.setattr(checker, "acomplete", model)
|
||||
monkeypatch.setattr(checker, "_capture_final_screen", AsyncMock(return_value=(None, "")))
|
||||
monkeypatch.setattr("artemis.controllers.unified_controller.get_driver", lambda *_: MagicMock())
|
||||
runner = FlashRunner(ctx, goal="核实双端布局并保存报告", max_turns=0)
|
||||
runner.executor = executor
|
||||
runner.summarizer = None
|
||||
runner._init_llm = MagicMock(return_value=MagicMock())
|
||||
runner._prepare_conversation = AsyncMock(return_value=(TranscriptLedger(), None, ""))
|
||||
runner._refresh_observation = AsyncMock(side_effect=AssertionError("note must not observe"))
|
||||
runner._android_observation_stale = True
|
||||
runner._invoke_model = AsyncMock(
|
||||
side_effect=[
|
||||
AIMessage(
|
||||
content="保存已观察结果。",
|
||||
tool_calls=[
|
||||
{
|
||||
"name": "save_note",
|
||||
"args": {"key": "findings", "content": "业务观察记录"},
|
||||
"id": "save-1",
|
||||
}
|
||||
],
|
||||
),
|
||||
AIMessage(
|
||||
content="提交报告。",
|
||||
tool_calls=[
|
||||
{
|
||||
"name": "report_task_status",
|
||||
"args": {"status": "completed", "explanation": report_body},
|
||||
"id": "done",
|
||||
}
|
||||
],
|
||||
),
|
||||
]
|
||||
)
|
||||
try:
|
||||
execution = await runner.run(State(initial_goal=runner.goal))
|
||||
await engine.shutdown()
|
||||
steps = engine.get_agent_friendly_steps()
|
||||
assert steps[0]["action_taken"]["action"] == "save_note"
|
||||
assert steps[0]["tool_calls"][0]["name"] == "save_note"
|
||||
assert not steps[0].get("pre_image_name") and not steps[0].get("post_image_name")
|
||||
runner._refresh_observation.assert_not_awaited()
|
||||
outcome = await flash.settle_flash_outcome(ctx, runner.goal, execution)
|
||||
model.assert_awaited_once()
|
||||
assert outcome["test_result"] == verdict.model_dump()
|
||||
actuator.execute.assert_not_called()
|
||||
finally:
|
||||
await engine.shutdown()
|
||||
@@ -22,6 +22,9 @@ def cross_target(monkeypatch):
|
||||
bindings={PC: ("pc", None)},
|
||||
tool_names={PC},
|
||||
previous_image=lambda *_: b"pc-before",
|
||||
# This fixture only returns a PC observation; stage-write guards have
|
||||
# dedicated coverage in test_flash_progress.py.
|
||||
is_read_only=lambda _: True,
|
||||
)
|
||||
monkeypatch.setattr("artemis.plugins.runtime.get_plugin_runtime", lambda _: plugins)
|
||||
runner = flash.FlashRunner.__new__(flash.FlashRunner)
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from artemis.agents.flash.progress import TaskProgress
|
||||
from artemis.agents.flash.runner import FlashRunner, _TurnRecord
|
||||
|
||||
|
||||
def plan(phone="active", pc="pending"):
|
||||
return {
|
||||
"reason": "按用户要求先手机页脚,再电脑独立检查",
|
||||
"stages": [
|
||||
{
|
||||
"id": "phone",
|
||||
"scope": "Android 主体、页脚及最后滚动",
|
||||
"resources": ["android"],
|
||||
"status": phone,
|
||||
"evidence": "步骤 12 页脚及 13 滚动截图" if phone in {"done", "blocked"} else "",
|
||||
},
|
||||
{
|
||||
"id": "pc",
|
||||
"scope": "电脑检查",
|
||||
"resources": ["cua-driver"],
|
||||
"status": pc,
|
||||
"evidence": "步骤 20 电脑截图" if pc == "done" else "",
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def test_ordered_stages_require_evidence_and_keep_outstanding_scope(tmp_path):
|
||||
progress = TaskProgress()
|
||||
progress.update(plan(), resources={"android", "cua-driver"}, directory=tmp_path)
|
||||
assert progress.guard("cua-driver")
|
||||
assert progress.guard("android") is None
|
||||
assert progress.guard(None) is None # read-only preflight
|
||||
assert progress.guard(None, completed=True)
|
||||
assert (tmp_path / "notes/task_progress.md").exists()
|
||||
with pytest.raises(ValidationError):
|
||||
progress.update(plan("pending", "active"), resources={"android", "cua-driver"})
|
||||
omitted = plan()
|
||||
omitted["stages"] = omitted["stages"][1:]
|
||||
with pytest.raises(ValueError, match="不能删除"):
|
||||
progress.update(omitted, resources={"android", "cua-driver"})
|
||||
progress.update(plan("done", "active"), resources={"android", "cua-driver"})
|
||||
assert progress.guard("cua-driver") is None
|
||||
assert progress.guard("android")
|
||||
progress.update(plan("done", "done"), resources={"android", "cua-driver"})
|
||||
assert progress.guard(None, completed=True) is None
|
||||
|
||||
|
||||
def test_pc_first_and_intentional_interleaving_are_supported():
|
||||
progress = TaskProgress()
|
||||
value = {
|
||||
"reason": "路由器调整后在手机验证",
|
||||
"stages": [
|
||||
{
|
||||
"id": "mixed",
|
||||
"scope": "电脑改参数,手机核对,循环验证",
|
||||
"resources": ["android", "cua-driver"],
|
||||
"status": "active",
|
||||
}
|
||||
],
|
||||
}
|
||||
progress.update(value, resources={"android", "cua-driver"})
|
||||
assert progress.guard("cua-driver") is None
|
||||
assert progress.guard("android") is None
|
||||
value["stages"][0]["resources"] = ["cua-driver"]
|
||||
progress.update(value, resources={"cua-driver"})
|
||||
assert progress.guard("cua-driver") is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_flash_preflight_does_not_authorize_pc_mutation(monkeypatch):
|
||||
plugins = SimpleNamespace(
|
||||
packages={"cua-driver": None},
|
||||
bindings={"plug_pc_look": ("cua-driver", None), "plug_pc_click": ("cua-driver", None)},
|
||||
tool_names={"plug_pc_look", "plug_pc_click"},
|
||||
is_read_only=lambda name: name == "plug_pc_look",
|
||||
)
|
||||
monkeypatch.setattr("artemis.plugins.runtime.get_plugin_runtime", lambda _: plugins)
|
||||
runner = FlashRunner.__new__(FlashRunner)
|
||||
runner.ctx = SimpleNamespace(data_engine=None, execution_target="android")
|
||||
runner.executor = SimpleNamespace(action_tool_names={"click"})
|
||||
runner._progress = TaskProgress()
|
||||
runner._progress.update(plan(), resources={"android", "cua-driver"})
|
||||
runner._execute_and_record_action = AsyncMock(return_value=(None, "", 0))
|
||||
runner._refresh_observation = AsyncMock(return_value=(None, ""))
|
||||
messages = []
|
||||
await runner._process_tool_calls(
|
||||
[
|
||||
{"name": "plug_pc_look", "id": "look", "args": {}},
|
||||
{"name": "plug_pc_click", "id": "click", "args": {}},
|
||||
],
|
||||
SimpleNamespace(indexed_elements=[]),
|
||||
messages,
|
||||
"预检",
|
||||
{},
|
||||
None,
|
||||
"",
|
||||
0,
|
||||
_TurnRecord(),
|
||||
)
|
||||
runner._execute_and_record_action.assert_awaited_once()
|
||||
assert runner._execute_and_record_action.call_args.args[0] == "plug_pc_look"
|
||||
assert messages[-1].tool_call_id == "click" and messages[-1].status == "error"
|
||||
runner._refresh_observation.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_final_report_reuses_exact_saved_body_and_rejects_missing_note(tmp_path):
|
||||
runner = FlashRunner.__new__(FlashRunner)
|
||||
runner.ctx = SimpleNamespace(
|
||||
data_engine=SimpleNamespace(base_dir=tmp_path, current_step_id="s", record_step=Mock())
|
||||
)
|
||||
runner.summarizer = None
|
||||
runner.final_report_received = False
|
||||
from artemis.utils.notes import save_verified_note
|
||||
|
||||
body = "# 报告\nPC 是深色;手机检查未全部覆盖。"
|
||||
save_verified_note(tmp_path, "report", body)
|
||||
messages = []
|
||||
args = {"status": "completed", "explanation": "全部完成", "report_note_key": "missing"}
|
||||
result = await runner._finalize_task_report(
|
||||
"report_task_status", args, "r", "", {}, None, "", messages
|
||||
)
|
||||
assert result is None and not runner.final_report_received
|
||||
args["report_note_key"] = "report"
|
||||
result = await runner._finalize_task_report(
|
||||
"report_task_status", args, "r2", "", {}, None, "", messages
|
||||
)
|
||||
assert result["explanation"] == body
|
||||
assert (
|
||||
runner.ctx.data_engine.record_step.call_args.kwargs["action_taken"]["args"]["explanation"]
|
||||
== body
|
||||
)
|
||||
@@ -99,6 +99,26 @@ def test_video_analyzer_bound_only_when_recording_tools_enabled(mock_context):
|
||||
HISTORY_TOOLS = {"search_history", "replay_steps", "get_step_screenshot"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("target", ["android", "computer"])
|
||||
def test_flash_binds_task_text_delivery_only_when_trace_available(mock_context, tmp_path, target):
|
||||
from types import SimpleNamespace
|
||||
|
||||
mock_context.execution_target = target
|
||||
with patch("artemis.controllers.unified_controller.get_driver"):
|
||||
runner = FlashRunner(mock_context, goal="写报告")
|
||||
tools = runner._get_tools()
|
||||
assert "save_note" not in {t.name for t in tools}
|
||||
assert "`save_note`" not in runner._render_system_prompt(tools)
|
||||
mock_context.data_engine = SimpleNamespace(base_dir=tmp_path)
|
||||
tools = runner._get_tools()
|
||||
names = {t.name for t in tools}
|
||||
assert {"save_note", "read_note", "list_notes"} <= names
|
||||
save = next(t for t in tools if t.name == "save_note")
|
||||
assert save.parameters["required"] == ["key", "content"]
|
||||
prompt = runner._render_system_prompt(tools)
|
||||
assert "notes/output.md" in prompt and "`save_note`" in prompt
|
||||
|
||||
|
||||
def test_history_tools_bound_only_with_a_data_engine_session(mock_context):
|
||||
with patch("artemis.controllers.unified_controller.get_driver"):
|
||||
# No DataEngine: nothing to read, the tools stay out (as in Pro).
|
||||
|
||||
@@ -127,23 +127,74 @@ async def test_completed_exploration_plan_does_not_force_an_extra_action(
|
||||
content="Confirmed home; recorded evidence.",
|
||||
tool_calls=[{"name": "save_note", "args": {}, "id": "plan"}],
|
||||
)
|
||||
invoke = AsyncMock(side_effect=[response, click()])
|
||||
final = AIMessage(content="# 完整调查报告\n已回主页,证据见步骤 3。未检查其他页面。")
|
||||
finishing = scenario in {"complete", "unchanged"}
|
||||
invoke = AsyncMock(side_effect=[response, final if finishing else click()])
|
||||
monkeypatch.setattr("artemis.agents.operator.operator.acomplete", invoke)
|
||||
messages = []
|
||||
result = await node._invoke_llm_loop(
|
||||
Mock(), messages, [SimpleNamespace(name="save_note")], [], state
|
||||
)
|
||||
if scenario == "complete":
|
||||
if finishing:
|
||||
assert invoke.await_count == 2
|
||||
assert result[0] is None
|
||||
assert final.content in result[1]
|
||||
assert state.execution_report == final.content
|
||||
assert any("完整报告正文" in m.content for m in messages if isinstance(m, SystemMessage))
|
||||
assert result[3] is False
|
||||
elif scenario == "pending":
|
||||
assert invoke.await_count == 1
|
||||
assert result[0] is None
|
||||
assert result[1] == "Confirmed home; recorded evidence."
|
||||
assert result[3] is False
|
||||
assert state.execution_report is None
|
||||
return
|
||||
else:
|
||||
assert invoke.await_count == 2
|
||||
assert result[0]
|
||||
assert state.execution_report is None
|
||||
assert any(isinstance(m, ToolMessage) and m.tool_call_id == "plan" for m in messages)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_final_report_is_discarded_when_guidance_arrives(operator, monkeypatch):
|
||||
node, state = operator
|
||||
state.execution_report = "旧报告"
|
||||
node.ctx.exploration_id = "exploration"
|
||||
node._read_task_plan_text = lambda: "- [x] 调查完成"
|
||||
pending = {"value": False}
|
||||
monkeypatch.setattr(
|
||||
"artemis.exploration.runtime.needs_exploration_turn", lambda ctx: pending["value"]
|
||||
)
|
||||
publish = Mock()
|
||||
monkeypatch.setattr("artemis.exploration.runtime.publish_response", publish)
|
||||
|
||||
async def answer(*args):
|
||||
pending["value"] = True
|
||||
return AIMessage(content="这是过时的最终报告")
|
||||
|
||||
monkeypatch.setattr("artemis.agents.operator.operator.acomplete", answer)
|
||||
result = await node._invoke_llm_loop(Mock(), [], [], [], state)
|
||||
assert result[0] is None
|
||||
assert state.execution_report is None
|
||||
publish.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_final_report_uses_bounded_recovery(operator, monkeypatch):
|
||||
node, state = operator
|
||||
node.ctx.exploration_id = "exploration"
|
||||
node._read_task_plan_text = lambda: "- [x] 调查完成"
|
||||
monkeypatch.setattr("artemis.exploration.runtime.needs_exploration_turn", lambda ctx: False)
|
||||
monkeypatch.setattr("artemis.exploration.runtime.publish_response", Mock())
|
||||
monkeypatch.setattr("artemis.agents.operator.operator.get_llm", Mock(return_value=Mock()))
|
||||
monkeypatch.setattr(settings, "ARTEMIS_TOOL_FALLBACK_MAX_NO_PROGRESS_ATTEMPTS", 2)
|
||||
invoke = AsyncMock(return_value=AIMessage(content=" "))
|
||||
monkeypatch.setattr("artemis.agents.operator.operator.acomplete", invoke)
|
||||
with pytest.raises(OperatorNoProgressError):
|
||||
await node._invoke_llm_loop(Mock(), [], [], [], state)
|
||||
assert state.execution_report is None
|
||||
assert invoke.await_count == 5
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_invalid_target_can_be_corrected_before_limit(operator, monkeypatch):
|
||||
result, messages, invoke = await run_loop(operator, monkeypatch, [click("14"), click("13")])
|
||||
|
||||
@@ -54,8 +54,8 @@ from artemis.memory.transcript import (
|
||||
|
||||
# SHA-256 snapshots of the legacy system message with the fixed inputs below.
|
||||
# Update these when an intentional template change alters the rendered prompt.
|
||||
GOLDEN_EMPTY_PLAN = "58bf034527b2d782f7746d8e379cb0decc44a05dfa18cb73153d49cd8265002b"
|
||||
GOLDEN_SENTINEL_PLAN = "b328052390a0a815c096b8ec4c5286421be487e7b6cd3bec55d6bc62a1f2518f"
|
||||
GOLDEN_EMPTY_PLAN = "e83c2c6cc1e28b04898e6602194cb2fbac67fe5a3aec8e109872c375f4e59944"
|
||||
GOLDEN_SENTINEL_PLAN = "dba4eb2cdcd9389148ac3228c6e9e69fdbe8d407dec76b58aa65509ad768c392"
|
||||
|
||||
SCREENSHOT_B64 = base64.b64encode(b"fake-jpeg-bytes").decode("utf-8")
|
||||
|
||||
|
||||
@@ -17,6 +17,7 @@ settlement timeout, when-semantics at the exit, run outcome, and the four
|
||||
switch-combination matrix."""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
@@ -83,7 +84,7 @@ async def test_exploration_question_prevents_completed_plan_from_starting_final_
|
||||
repo.ask(run["id"], "下一步做什么?", "返回主页", "ask")
|
||||
with patch("artemis.graph.graph.run_final_check", new=AsyncMock()) as reviewer:
|
||||
assert convergence_gate(_make_state(), ctx) == "continue"
|
||||
result = await exit_settlement_node(_make_state(), ctx)
|
||||
result = await exit_settlement_node(_make_state(execution_report="本次调查报告"), ctx)
|
||||
assert result == {"exit_settlement_route": "continue"}
|
||||
reviewer.assert_not_called()
|
||||
assert repo.get(session["id"])["pending_question"]
|
||||
@@ -100,7 +101,7 @@ async def test_exploration_late_return_home_guidance_invalidates_final_review(ex
|
||||
return CheckReport(verdicts=[], unmet_subgoals=[])
|
||||
|
||||
with patch("artemis.graph.graph.run_final_check", new=review):
|
||||
result = await exit_settlement_node(_make_state(), ctx)
|
||||
result = await exit_settlement_node(_make_state(execution_report="本次调查报告"), ctx)
|
||||
assert result == {"exit_settlement_route": "continue"}
|
||||
assert repo.run(run["id"])["status"] == "running"
|
||||
assert repo.run(run["id"])["user_guidance_count"] == 1
|
||||
@@ -118,12 +119,99 @@ async def test_review_stops_capture_before_waiting_and_seals_exploration(explora
|
||||
return CheckReport(verdicts=[], unmet_subgoals=[])
|
||||
|
||||
with patch("artemis.graph.graph.run_final_check", new=review):
|
||||
result = await exit_settlement_node(_make_state(), ctx)
|
||||
result = await exit_settlement_node(_make_state(execution_report="本次调查报告"), ctx)
|
||||
assert result["exit_settlement_route"] == "end"
|
||||
assert repo.get(session["id"])["status"] == "finalizing"
|
||||
ctx.on_execution_resume.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_completed_exploration_waits_for_report_before_review(exploration_exit):
|
||||
ctx, repo, session, run = exploration_exit
|
||||
state = State.initial("保存并交付完整报告")
|
||||
with patch("artemis.graph.graph.run_final_check", new=AsyncMock()) as review:
|
||||
assert convergence_gate(state, ctx) == "continue"
|
||||
assert await exit_settlement_node(state, ctx) == {"exit_settlement_route": "continue"}
|
||||
review.assert_not_called()
|
||||
ctx.on_execution_review.assert_not_called()
|
||||
assert repo.run(run["id"])["status"] == "running"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("verdict_status", ["passed", "failed", "inconclusive"])
|
||||
async def test_report_is_saved_and_fully_delivered_before_independent_review(
|
||||
exploration_exit, verdict_status
|
||||
):
|
||||
from artemis.runtime.case_result import CaseTestResult
|
||||
|
||||
ctx, repo, session, run = exploration_exit
|
||||
body = "# 本次调查报告\n" + "实际证据及未测范围。" * 2400
|
||||
state = State(initial_goal="交付完整报告", execution_report=body)
|
||||
_write_plan(ctx.data_engine.base_dir, "- [x] 调查并交付报告\n - assert: 实际页面结果\n")
|
||||
|
||||
async def review(*args, **kwargs):
|
||||
directory = ctx.data_engine.base_dir
|
||||
assert (directory / "notes" / "output.md").read_text(encoding="utf-8") == body
|
||||
assert json.loads((directory / "result.json").read_text(encoding="utf-8")) == {
|
||||
"result": body
|
||||
}
|
||||
delivered = [m for m in repo.messages(session["id"]) if m.get("kind") == "execution_report"]
|
||||
assert len(delivered) == 3
|
||||
assert "".join(m["text"] for m in delivered) == body
|
||||
assert kwargs["execution_report"] == body
|
||||
assert kwargs["report_delivery"]["exploration_message_sequences"] == [
|
||||
m["sequence"] for m in delivered
|
||||
]
|
||||
return CheckReport(
|
||||
verdicts=[
|
||||
CheckVerdict(
|
||||
item_text="实际页面结果",
|
||||
kind="assert",
|
||||
status=verdict_status,
|
||||
evidence="步骤 1",
|
||||
)
|
||||
],
|
||||
test_result=CaseTestResult(
|
||||
status=verdict_status, reason="按本次实际证据裁定", evidence=["步骤 1 实际页面"]
|
||||
),
|
||||
)
|
||||
|
||||
with patch("artemis.graph.graph.run_final_check", new=review):
|
||||
result = await exit_settlement_node(state, ctx)
|
||||
assert result["run_outcome"]["test_result"]["status"] == verdict_status
|
||||
assert repo.run(run["id"])["status"] == "finalizing"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_report_delivery_is_idempotent_with_review_disabled(exploration_exit):
|
||||
ctx, repo, session, run = exploration_exit
|
||||
ctx.execution_setup.disable_checker = True
|
||||
state = State(initial_goal="调查", execution_report="完整报告正文")
|
||||
assert convergence_gate(state, ctx) == "exit_settlement"
|
||||
with patch("artemis.graph.graph.run_final_check", new=AsyncMock()) as review:
|
||||
await exit_settlement_node(state, ctx)
|
||||
await exit_settlement_node(state, ctx)
|
||||
review.assert_not_called()
|
||||
delivered = [m for m in repo.messages(session["id"]) if m.get("kind") == "execution_report"]
|
||||
assert len(delivered) == 1
|
||||
assert delivered[0]["text"] == "完整报告正文"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_report_write_failure_cannot_release_as_completed(exploration_exit, monkeypatch):
|
||||
from pathlib import Path
|
||||
|
||||
ctx, repo, session, run = exploration_exit
|
||||
state = State(initial_goal="调查", execution_report="完整报告正文")
|
||||
monkeypatch.setattr(Path, "replace", MagicMock(side_effect=OSError("disk full")))
|
||||
with patch("artemis.graph.graph.run_final_check", new=AsyncMock()) as review:
|
||||
with pytest.raises(OSError, match="disk full"):
|
||||
await exit_settlement_node(state, ctx)
|
||||
review.assert_not_called()
|
||||
assert repo.run(run["id"])["status"] == "running"
|
||||
assert not list(ctx.data_engine.base_dir.rglob("*.tmp"))
|
||||
|
||||
|
||||
def _make_ctx(tmp_path, **setup_kwargs):
|
||||
setup_kwargs.setdefault("disable_checker", False)
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
@@ -311,7 +399,16 @@ async def test_final_check_receives_ledger_and_at_end_items(tmp_path):
|
||||
|
||||
captured = {}
|
||||
|
||||
async def fake_final(ctx_arg, goal, plan_text, ledger, check_items, attempt_id=None):
|
||||
async def fake_final(
|
||||
ctx_arg,
|
||||
goal,
|
||||
plan_text,
|
||||
ledger,
|
||||
check_items,
|
||||
attempt_id=None,
|
||||
execution_report=None,
|
||||
report_delivery=None,
|
||||
):
|
||||
captured["goal"] = goal
|
||||
captured["ledger"] = ledger
|
||||
captured["items"] = list(check_items)
|
||||
|
||||
@@ -66,7 +66,15 @@ def shared_mcp_backend(tmp_path, monkeypatch):
|
||||
key: registration.info[key] for key in ("protocol", "installation", "instance", "pid")
|
||||
}
|
||||
|
||||
server = uvicorn.Server(uvicorn.Config(app, host="127.0.0.1", port=port, log_level="error"))
|
||||
server = uvicorn.Server(
|
||||
uvicorn.Config(
|
||||
app,
|
||||
host="127.0.0.1",
|
||||
port=port,
|
||||
log_level="error",
|
||||
timeout_graceful_shutdown=5, # Match the user-started service's SSE shutdown budget.
|
||||
)
|
||||
)
|
||||
thread = threading.Thread(target=server.run, kwargs={"sockets": [listener]}, daemon=True)
|
||||
thread.start()
|
||||
deadline = time.monotonic() + 8
|
||||
|
||||
@@ -48,6 +48,46 @@ def test_helper_tools_are_not_device_actions():
|
||||
assert not (AGENT_TOOL_NAMES & executor.action_tool_names)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_flash_saves_and_reads_text_with_verified_receipt_without_device_actions(tmp_path):
|
||||
import hashlib
|
||||
import json
|
||||
|
||||
executor = _make_executor(agent_name="flash")
|
||||
executor.ctx.data_engine = Mock(base_dir=tmp_path, current_step_id=None)
|
||||
state = _state()
|
||||
result = await executor.execute(
|
||||
"save_note", {"key": "reusable_case", "content": "# 用例\n检查标题"}, "save", state
|
||||
)
|
||||
assert result.status == "success"
|
||||
receipt = json.loads(result.text_summary)
|
||||
data = (tmp_path / receipt["path"]).read_bytes()
|
||||
assert receipt["verified"] is True and receipt["sha256"] == hashlib.sha256(data).hexdigest()
|
||||
assert receipt["bytes"] == len(data)
|
||||
read = await executor.execute("read_note", {"key": "reusable_case"}, "read", state)
|
||||
assert read.status == "success" and "检查标题" in read.text_summary
|
||||
assert result.screenshot_bytes is None and result.ui_elements_text is None
|
||||
executor.actuator.execute.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("key", ["../outside", "C:\\outside", "output", "output.MD"])
|
||||
async def test_flash_note_rejects_paths_and_reserved_final_report(tmp_path, key):
|
||||
executor = _make_executor(agent_name="flash")
|
||||
executor.ctx.data_engine = Mock(base_dir=tmp_path, current_step_id=None)
|
||||
result = await executor.execute("save_note", {"key": key, "content": "data"}, "save", _state())
|
||||
assert result.status == "error"
|
||||
assert not (tmp_path / "notes").exists()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_flash_note_does_not_fall_back_to_working_directory_without_trace():
|
||||
result = await _make_executor(agent_name="flash").execute(
|
||||
"save_note", {"key": "report", "content": "data"}, "save", _state()
|
||||
)
|
||||
assert result.status == "error" and "trace" in result.text_summary
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ask_explorer
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -56,7 +56,9 @@ SCOPED_PROMPT_FILES = [
|
||||
# apply_operator_prompt_contract. Anything new landing here should be a conscious
|
||||
# decision.
|
||||
#: Backticked identifiers in prompts that are parameters or note keys, not tools.
|
||||
NON_TOOL_IDENTIFIERS = frozenset({"analysis", "task_plan", "tool_enum", "target_description"})
|
||||
NON_TOOL_IDENTIFIERS = frozenset(
|
||||
{"analysis", "task_plan", "tool_enum", "target_description", "report_note_key"}
|
||||
)
|
||||
|
||||
_TOOL_REF = re.compile(r"(?:`([a-z][a-z0-9_]{2,})`|\b([a-z][a-z0-9_]{2,})\()")
|
||||
|
||||
|
||||
@@ -245,6 +245,45 @@ async def test_executor_cancellation_reaches_coroutine():
|
||||
await executor.stop()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_tool_runs_once_off_both_event_loops():
|
||||
import threading
|
||||
|
||||
executor = ToolExecutor()
|
||||
executor.start()
|
||||
caller = threading.get_ident()
|
||||
calls = []
|
||||
|
||||
def operation(value):
|
||||
calls.append(threading.get_ident())
|
||||
return {"accepted": value}
|
||||
|
||||
try:
|
||||
assert await executor.invoke(operation, "job") == {"accepted": "job"}
|
||||
assert len(calls) == 1
|
||||
assert calls[0] not in {caller, executor.thread.ident}
|
||||
finally:
|
||||
await executor.stop()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sync_tool_error_does_not_repeat_the_side_effect():
|
||||
executor = ToolExecutor()
|
||||
executor.start()
|
||||
calls = []
|
||||
|
||||
def operation():
|
||||
calls.append("submitted")
|
||||
raise ValueError("reply lost")
|
||||
|
||||
try:
|
||||
with pytest.raises(ValueError, match="reply lost"):
|
||||
await executor.invoke(operation)
|
||||
assert calls == ["submitted"]
|
||||
finally:
|
||||
await executor.stop()
|
||||
|
||||
|
||||
def test_stale_pid_and_wrong_installation_are_not_reused(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(tempfile, "tempdir", str(tmp_path))
|
||||
directory = registry.registry_directory()
|
||||
|
||||
@@ -0,0 +1,55 @@
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from apps.admin_console.services.automation_store import AutomationStore
|
||||
from mcp_server import task_requests
|
||||
from mcp_server.tools import task_runner
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def receipts(tmp_path, monkeypatch):
|
||||
store = AutomationStore(tmp_path / "automation.db")
|
||||
monkeypatch.setattr(task_requests, "store", store)
|
||||
return store
|
||||
|
||||
|
||||
def test_concurrent_retries_dispatch_once(receipts):
|
||||
execute = Mock(side_effect=lambda sid: {"trace_id": sid, "status": "queued"})
|
||||
with ThreadPoolExecutor(max_workers=4) as pool:
|
||||
results = list(
|
||||
pool.map(
|
||||
lambda _: task_requests.run_once("one", {"goal": "observe"}, execute), range(8)
|
||||
)
|
||||
)
|
||||
assert execute.call_count == 1
|
||||
assert len({result["trace_id"] for result in results}) == 1
|
||||
assert task_requests.lookup("one")["status"] == "queued"
|
||||
receipts.save_job({"session_id": results[0]["trace_id"], "status": "completed", "position": 1})
|
||||
assert task_requests.lookup("one")["status"] == "completed"
|
||||
assert task_requests.lookup("one")["status_scope"] == "task_queue"
|
||||
|
||||
|
||||
def test_conflicting_content_and_lost_reply_never_repeat(receipts):
|
||||
execute = Mock(side_effect=ConnectionError("after admission"))
|
||||
with pytest.raises(ConnectionError):
|
||||
task_requests.run_once("one", {"goal": "observe"}, execute)
|
||||
receipt = task_requests.run_once("one", {"goal": "observe"}, execute)
|
||||
assert receipt["submission_state"] == "unknown"
|
||||
assert receipt["trace_id"]
|
||||
assert execute.call_count == 1
|
||||
with pytest.raises(ValueError, match="不同任务内容"):
|
||||
task_requests.run_once("one", {"goal": "write"}, execute)
|
||||
assert task_requests.lookup("missing")["status"] == "not_found"
|
||||
|
||||
|
||||
def test_public_task_entry_uses_receipts_before_dispatch(receipts, monkeypatch):
|
||||
execute = Mock(
|
||||
side_effect=lambda **kwargs: {"trace_id": kwargs["trace_id"], "status": "queued"}
|
||||
)
|
||||
monkeypatch.setattr(task_runner, "_run_task", execute)
|
||||
first = task_runner.mobile_run_task("observe", request_id="public-one")
|
||||
second = task_runner.mobile_run_task("observe", request_id="public-one")
|
||||
assert first["trace_id"] == second["trace_id"]
|
||||
execute.assert_called_once()
|
||||
@@ -72,6 +72,49 @@ def _step(i: int, subgoal: str = "hash-a", **overrides) -> dict:
|
||||
return step
|
||||
|
||||
|
||||
@pytest.mark.parametrize("minimal", [False, True])
|
||||
def test_runtime_file_paths_survive_even_minimal_history_folding(minimal):
|
||||
from artemis.memory.chunking import build_action_ledger
|
||||
|
||||
action = {
|
||||
"action": "plug_pc_save",
|
||||
"plugin_id": "cua-driver",
|
||||
"attachments": [
|
||||
{"path": "artifacts/plugins/cua-driver/footer.png", "sha256": "a" * 64, "bytes": 123}
|
||||
],
|
||||
}
|
||||
text = build_action_ledger([_step(35, action_taken=action)], SESSION_START, minimal=minimal)
|
||||
assert "Step 35" in text and "footer.png" in text and "a" * 64 in text
|
||||
|
||||
|
||||
def test_default_larger_batches_reduce_capsule_calls_without_losing_step_index():
|
||||
from artemis.config import MemoryChunkingConfig
|
||||
|
||||
current = MemoryChunkingConfig()
|
||||
steps = [_step(i) for i in range(1, 48)]
|
||||
counts = []
|
||||
for minimum, target in [(3, 2000), (current.min_steps, current.target_source_tokens)]:
|
||||
ledger, chunker, _, capsule = _make(
|
||||
steps,
|
||||
min_active=5,
|
||||
min_steps=minimum,
|
||||
target_tokens=target,
|
||||
auto_capsule=True,
|
||||
meter=lambda: 75_000,
|
||||
)
|
||||
_run_turns(ledger, chunker, 1, 47, lambda _: "same-stage")
|
||||
counts.append(len(capsule.submitted))
|
||||
consumed = {sid for c in chunker.chunks for sid in c.source_step_ids}
|
||||
consumed.update(
|
||||
sid for era in chunker.eras for c in era.chunks for sid in c.source_step_ids
|
||||
)
|
||||
remaining = {sid for turn in ledger.unchunked_turns() for sid in turn.get("step_keys", [])}
|
||||
assert consumed | remaining # remaining live/frozen records are not dropped
|
||||
for number in range(1, 42):
|
||||
assert f"s{number}" in consumed | remaining
|
||||
assert counts[1] < counts[0]
|
||||
|
||||
|
||||
class FakeEngine:
|
||||
def __init__(self, steps: list[dict], base_dir=None):
|
||||
self.steps = steps
|
||||
|
||||
@@ -249,7 +249,7 @@ def test_cli_mcp_install_codex_preserves_config_and_is_idempotent(tmp_path, monk
|
||||
config_file.write_text(
|
||||
'model = "test-model"\n\n'
|
||||
'[mcp_servers.existing]\ncommand = "echo"\n\n'
|
||||
'[mcp_servers.artemis]\ncommand = "old-command"\n',
|
||||
'[mcp_servers.artemis]\ncommand = "old-command"\nargs = ["-m", "mcp_server"]\n',
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
@@ -283,12 +283,12 @@ def test_cli_mcp_install_codex_preserves_config_and_is_idempotent(tmp_path, monk
|
||||
"mobile_export_report",
|
||||
]
|
||||
assert data["mcp_servers"]["artemis"]["env"]["PYTHONPATH"] == str(tmp_path)
|
||||
assert config_text.count("# BEGIN ARTEMIS MCP CONFIG") == 1
|
||||
assert config_text.count("[mcp_servers.artemis]") == 1
|
||||
|
||||
agents_file = codex_dir / "AGENTS.md"
|
||||
agents_text = agents_file.read_text(encoding="utf-8")
|
||||
assert "Mobile Testing Mindset" in agents_text
|
||||
assert agents_text.count("<!-- BEGIN ARTEMIS MOBILE TESTING RULES -->") == 1
|
||||
assert not agents_file.exists()
|
||||
skill_file = tmp_path / ".agents/skills/artemis-use/SKILL.md"
|
||||
assert "name: artemis-use" in skill_file.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_cli_restart_help():
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.integrations import codex_skill as integration
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def bundle(tmp_path, monkeypatch):
|
||||
root = tmp_path / "source"
|
||||
source = root / "skills" / "artemis-use"
|
||||
source.mkdir(parents=True)
|
||||
(source / "SKILL.md").write_text(
|
||||
"---\nname: artemis-use\ndescription: official\n---\n", "utf-8"
|
||||
)
|
||||
(source / "references").mkdir()
|
||||
(source / "references" / "modes.md").write_text("official modes", "utf-8")
|
||||
codex_home = tmp_path / "用户配置" / ".codex"
|
||||
monkeypatch.setenv("CODEX_HOME", str(codex_home))
|
||||
return root, source, codex_home
|
||||
|
||||
|
||||
def test_install_overwrites_entire_reserved_directory_and_is_noop_when_unchanged(bundle):
|
||||
root, source, home = bundle
|
||||
target = home / "skills" / "artemis-use"
|
||||
target.mkdir(parents=True)
|
||||
(target / "SKILL.md").write_text("unmanaged custom content", "utf-8")
|
||||
(target / "custom.txt").write_text("must be removed", "utf-8")
|
||||
unrelated = home / "skills" / "personal" / "SKILL.md"
|
||||
unrelated.parent.mkdir()
|
||||
unrelated.write_text("personal", "utf-8")
|
||||
assert integration.sync_official_skill(root)["status"] == "updated"
|
||||
assert not (target / "custom.txt").exists()
|
||||
assert (target / "SKILL.md").read_bytes() == (source / "SKILL.md").read_bytes()
|
||||
assert unrelated.read_text("utf-8") == "personal"
|
||||
before = (target / "SKILL.md").stat().st_mtime_ns
|
||||
assert integration.sync_official_skill(root)["status"] == "unchanged"
|
||||
assert (target / "SKILL.md").stat().st_mtime_ns == before
|
||||
(target / "SKILL.md").write_text("edited", "utf-8")
|
||||
assert integration.sync_official_skill(root)["status"] == "updated"
|
||||
assert (target / "SKILL.md").read_bytes() == (source / "SKILL.md").read_bytes()
|
||||
|
||||
|
||||
def test_update_removes_retired_files_and_migrates_only_marked_rules(bundle):
|
||||
root, source, home = bundle
|
||||
integration.sync_official_skill(root)
|
||||
original = "user prefix\n<!-- BEGIN ARTEMIS MOBILE TESTING RULES -->\nold\n<!-- END ARTEMIS MOBILE TESTING RULES -->\nuser suffix\n"
|
||||
for name in ("AGENTS.md", "AGENTS.override.md"):
|
||||
(home / name).write_text(original, "utf-8")
|
||||
(source / "references" / "modes.md").unlink()
|
||||
integration.sync_official_skill(root)
|
||||
assert not (home / "skills/artemis-use/references/modes.md").exists()
|
||||
for name in ("AGENTS.md", "AGENTS.override.md"):
|
||||
assert (home / name).read_text("utf-8") == "user prefix\nuser suffix\n"
|
||||
assert (home / (name + ".artemis-update.bak")).read_text("utf-8") == original
|
||||
|
||||
|
||||
def test_failed_replacement_restores_previous_bundle(bundle, monkeypatch):
|
||||
root, source, home = bundle
|
||||
integration.sync_official_skill(root)
|
||||
target = home / "skills/artemis-use"
|
||||
original = (target / "SKILL.md").read_bytes()
|
||||
(source / "SKILL.md").write_text("new version", "utf-8")
|
||||
rename = Path.rename
|
||||
|
||||
def fail_staging(path, destination):
|
||||
if path.name.startswith(".artemis-use-stage-"):
|
||||
raise PermissionError("held open")
|
||||
return rename(path, destination)
|
||||
|
||||
monkeypatch.setattr(Path, "rename", fail_staging)
|
||||
assert integration.sync_skill_safely(root)["status"] == "error"
|
||||
assert (target / "SKILL.md").read_bytes() == original
|
||||
assert not list(target.parent.glob(".artemis-use-stage-*"))
|
||||
|
||||
|
||||
def test_newer_official_bundle_is_not_downgraded(bundle):
|
||||
root, _, home = bundle
|
||||
integration.sync_official_skill(root)
|
||||
stamp = home / "skills/artemis-use" / integration.STAMP
|
||||
data = json.loads(stamp.read_text("utf-8"))
|
||||
data["artemis_version"] = "99.0.0"
|
||||
stamp.write_text(json.dumps(data), "utf-8")
|
||||
assert integration.sync_official_skill(root)["status"] == "newer_installed"
|
||||
|
||||
|
||||
def test_default_directory_and_duplicate_cleanup(bundle, tmp_path, monkeypatch):
|
||||
root, source, _ = bundle
|
||||
monkeypatch.delenv("CODEX_HOME")
|
||||
monkeypatch.setattr(Path, "home", lambda: tmp_path)
|
||||
legacy = tmp_path / ".codex/skills/artemis-use"
|
||||
legacy.mkdir(parents=True)
|
||||
(legacy / "SKILL.md").write_text("old", "utf-8")
|
||||
assert integration.sync_official_skill(root)["path"] == str(legacy)
|
||||
modern = tmp_path / ".agents/skills/artemis-use"
|
||||
modern.mkdir(parents=True)
|
||||
(modern / "SKILL.md").write_text("duplicate", "utf-8")
|
||||
assert integration.sync_official_skill(root)["path"] == str(modern)
|
||||
assert not legacy.exists()
|
||||
|
||||
|
||||
def test_actual_bundle_links_are_self_contained():
|
||||
import re
|
||||
|
||||
source = Path(__file__).resolve().parents[2] / "skills/artemis-use"
|
||||
for document in source.rglob("*.md"):
|
||||
for relative in re.findall(r"\]\(([^)]+)\)", document.read_text("utf-8")):
|
||||
if "://" in relative or relative.startswith("#"):
|
||||
continue
|
||||
target = (document.parent / relative.split("#", 1)[0]).resolve()
|
||||
assert target.is_relative_to(source.resolve()), (document, relative)
|
||||
assert target.exists(), (document, relative)
|
||||
|
||||
|
||||
def test_malformed_stamp_does_not_block_authoritative_update(bundle):
|
||||
root, source, home = bundle
|
||||
integration.sync_official_skill(root)
|
||||
target = home / "skills/artemis-use"
|
||||
(target / integration.STAMP).write_text("[]", "utf-8")
|
||||
(target / "SKILL.md").write_text("unofficial", "utf-8")
|
||||
assert integration.sync_official_skill(root)["status"] == "updated"
|
||||
assert (target / "SKILL.md").read_bytes() == (source / "SKILL.md").read_bytes()
|
||||
@@ -0,0 +1,39 @@
|
||||
"""The default SDK path must return the answer already prepared for review."""
|
||||
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.config import OutputConfig
|
||||
from artemis.graph.state import State
|
||||
from artemis.sdk.agent import Agent
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("outputter_enabled", [True, False])
|
||||
async def test_default_output_reuses_executor_report_without_another_model(
|
||||
tmp_path, monkeypatch, outputter_enabled
|
||||
):
|
||||
synthesize = AsyncMock()
|
||||
monkeypatch.setattr("artemis.sdk.agent.outputter", synthesize)
|
||||
ctx = SimpleNamespace(
|
||||
execution_setup=SimpleNamespace(
|
||||
outputter=SimpleNamespace(enabled=outputter_enabled, force_synthesis=False),
|
||||
disable_outputter=not outputter_enabled,
|
||||
)
|
||||
)
|
||||
result_path = tmp_path / "result.json"
|
||||
body = "# 调查结果\n双端检查已完成;未测功能不能记为通过。"
|
||||
result = await Agent._extract_output(
|
||||
object.__new__(Agent),
|
||||
task_name="exploration",
|
||||
ctx=ctx,
|
||||
request=SimpleNamespace(llm_output_path=result_path),
|
||||
output_config=OutputConfig(structured_output=None, output_description=None),
|
||||
state=State(initial_goal="调查", execution_report=body),
|
||||
)
|
||||
assert result == body
|
||||
assert json.loads(result_path.read_text(encoding="utf-8")) == {"result": body}
|
||||
synthesize.assert_not_called()
|
||||
@@ -232,6 +232,38 @@ async def test_rerun_includes_user_edited_preconditions(service, monkeypatch):
|
||||
assert "已连接测试网络" in enqueue.call_args.args[0].goal
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("field", ["goal", "preconditions"])
|
||||
async def test_unresolved_case_never_reserves_run_or_creates_batch(service, monkeypatch, field):
|
||||
from apps.admin_console.services import automation_service
|
||||
|
||||
enqueue = AsyncMock()
|
||||
monkeypatch.setattr("apps.admin_console.routers.tasks.run_task", enqueue)
|
||||
monkeypatch.setattr(automation_service, "store", service.repository.database)
|
||||
session = service.create(CreateExploration(goal="探索", request_id="create"))
|
||||
draft = {"name": "旧版用例", "goal": "检查页面", field: "报告保存到 ${报告位置}"}
|
||||
service.repository.save_case(session["id"], CaseDraft(**draft), 0, "save")
|
||||
with pytest.raises(ValueError, match=r"未填写参数.*\$\{报告位置\}"):
|
||||
await service.validate(session["id"], ValidateCaseRequest(request_id="validate", version=1))
|
||||
with pytest.raises(ValueError, match="未填写参数"):
|
||||
service.create_case_batch(session["id"], CaseBatchRequest(request_id="batch", version=1))
|
||||
enqueue.assert_not_called()
|
||||
assert service.repository.runs(session["id"]) == []
|
||||
assert service.repository.database.records("batch") == []
|
||||
assert service.repository.case(session["id"])["version"] == 1
|
||||
|
||||
# A saved correction can use the same admission request ID; rejection created no run.
|
||||
service.repository.save_case(
|
||||
session["id"],
|
||||
CaseDraft(name="可执行", goal="检查页面,报告保存到本次任务 trace"),
|
||||
1,
|
||||
"edit",
|
||||
)
|
||||
enqueue.return_value = {"status": "queued"}
|
||||
await service.validate(session["id"], ValidateCaseRequest(request_id="validate", version=2))
|
||||
enqueue.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("profile", ["flash", "pro"])
|
||||
def test_case_batch_creates_only_an_idempotent_draft_with_bound_resources(
|
||||
service, monkeypatch, profile
|
||||
|
||||
@@ -82,6 +82,50 @@ async def test_generation_is_idempotent_and_preserves_rerun_independence(synthes
|
||||
model.with_structured_output.return_value.ainvoke.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_synthesis_repairs_unbound_parameters_once_with_original_evidence(synthesis):
|
||||
repository, session, run, model = synthesis
|
||||
invoke = model.with_structured_output.return_value.ainvoke
|
||||
invoke.side_effect = [
|
||||
CaseDraft(name="错误草稿", goal="访问 ${首页URL}", preconditions="已指定 ${报告位置}"),
|
||||
CaseDraft(name="修订", goal="核对设置,报告保存在本次任务 trace。", evidence=["步骤3"]),
|
||||
]
|
||||
result = await generate_case(
|
||||
repository, session["id"], "generate", run["id"], 0, ctx=SimpleNamespace(task_control=None)
|
||||
)
|
||||
assert result["version"] == 1 and "${" not in result["goal"]
|
||||
assert invoke.await_count == 2
|
||||
messages = invoke.await_args.args[0]
|
||||
assert "不生成 ${名称}" in messages[0].content
|
||||
assert "用户明确指定的位置必须保留" in messages[0].content
|
||||
assert "不要让一次复跑递归地产生下一份复跑用例" in messages[0].content
|
||||
assert "保留该业务验收" in messages[0].content
|
||||
assert "设置开启" in messages[1].content
|
||||
assert "不得猜测未知业务输入或降低验收标准" in messages[-1].content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failed_parameter_repair_never_saves_unexecutable_generated_case(synthesis):
|
||||
repository, session, run, model = synthesis
|
||||
repository.save_case(session["id"], CaseDraft(name="原用例", goal="核对设置"), 0, "save")
|
||||
invoke = model.with_structured_output.return_value.ainvoke
|
||||
invoke.return_value = CaseDraft(name="错误", goal="保存到 ${报告位置}")
|
||||
with pytest.raises(ValueError, match="未填写参数"):
|
||||
await generate_case(
|
||||
repository,
|
||||
session["id"],
|
||||
"generate",
|
||||
run["id"],
|
||||
1,
|
||||
ctx=SimpleNamespace(task_control=None),
|
||||
)
|
||||
assert invoke.await_count == 2
|
||||
assert repository.case(session["id"])["version"] == 1
|
||||
assert repository.case(session["id"])["name"] == "原用例"
|
||||
assert repository.database.records("exploration_generation")[0]["status"] == "failed"
|
||||
assert repository.run(run["id"])["outcome"]["test_result"]["status"] == "passed"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_model_generation_cannot_overwrite_a_concurrent_manual_revision(synthesis):
|
||||
repository, session, run, model = synthesis
|
||||
|
||||
@@ -132,7 +132,7 @@ command = "leave-alone"
|
||||
assert config["disabled_tools"] == ["mobile_config"]
|
||||
assert config["tool_timeout_sec"] == 900
|
||||
assert config["env"]["OPENAI_API_KEY"] == "test-only-secret"
|
||||
assert config["enabled_tools"] == ["mobile_run_task", "mobile_batch"]
|
||||
assert config["enabled_tools"] == ["mobile_run_task"]
|
||||
assert data["mcp_servers"]["other"]["command"] == "leave-alone"
|
||||
assert not update.refresh_codex(path, root, "new-python", ("mobile_run_task", "mobile_batch"))
|
||||
assert path.with_name(path.name + ".artemis-update.bak").read_text(encoding="utf-8") == original
|
||||
@@ -178,7 +178,7 @@ def test_codex_sync_preserves_tool_policies_verbatim(tmp_path, monkeypatch, poli
|
||||
"PYTHONUNBUFFERED": "1",
|
||||
"ARTEMIS_DESKTOP_NOTIFY": "true",
|
||||
}
|
||||
assert entry["enabled_tools"] == list(mcp_command.ARTEMIS_MCP_TOOLS)
|
||||
assert entry["enabled_tools"] == ["mobile_run_task"]
|
||||
assert entry["disabled_tools"] == ["mobile_config"]
|
||||
assert entry["tools"] == before["mcp_servers"]["artemis"]["tools"]
|
||||
assert policies.split("\n", 1)[1] in refreshed
|
||||
@@ -199,7 +199,10 @@ def test_codex_sync_reports_generated_toml_error_without_blame_or_overwrite(
|
||||
path.write_text(original, encoding="utf-8")
|
||||
monkeypatch.setenv("CODEX_HOME", str(codex_home))
|
||||
monkeypatch.setenv("ARTEMIS_LAUNCH_LANG", "zh-CN")
|
||||
monkeypatch.setattr(update, "_replace_toml_tables", lambda *_: 'tools = {"secret": true}\n')
|
||||
monkeypatch.setattr(
|
||||
"artemis.integrations.codex_mcp._replace_toml_tables",
|
||||
lambda *_: 'tools = {"secret": true}\n',
|
||||
)
|
||||
assert update.refresh_mcp(tmp_path / "project") == 1
|
||||
assert path.read_text(encoding="utf-8") == original
|
||||
assert not path.with_name(path.name + ".artemis-update.bak").exists()
|
||||
@@ -252,7 +255,7 @@ def test_mcp_sync_updates_only_codex_and_preserves_other_clients(tmp_path, monke
|
||||
assert entry["enabled"] is False
|
||||
assert entry["env"]["OPENAI_API_KEY"] == "keep-secret"
|
||||
assert entry["env"]["PYTHONPATH"] == str(root)
|
||||
assert entry["enabled_tools"] == list(mcp_command.ARTEMIS_MCP_TOOLS)
|
||||
assert entry["enabled_tools"] == ["mobile_run_task", "retired_tool"]
|
||||
cursor = json.loads((cursor_home / "mcp.json").read_text(encoding="utf-8"))
|
||||
assert "artemis" not in cursor["mcpServers"]
|
||||
assert cursor["mcpServers"]["other"] == {"command": "keep"}
|
||||
@@ -331,9 +334,9 @@ def test_codex_sync_reports_missing_directory_without_touching_other_clients(
|
||||
assert update.refresh_mcp(tmp_path / "project") == 0
|
||||
output = capsys.readouterr().out
|
||||
assert str(tmp_path / ".codex/config.toml") in output
|
||||
assert "未发现 Codex 配置目录" in output
|
||||
assert "--install codex" in output
|
||||
assert not (tmp_path / ".codex").exists()
|
||||
assert "已安装" in output
|
||||
assert (tmp_path / ".codex/config.toml").is_file()
|
||||
assert (tmp_path / ".agents/skills/artemis-use/SKILL.md").is_file()
|
||||
|
||||
|
||||
def test_custom_codex_command_is_preserved_and_reported(tmp_path, monkeypatch, capsys):
|
||||
|
||||
@@ -319,7 +319,8 @@ async def test_mid_stream_failure_records_stream_reset_payload(monkeypatch):
|
||||
assert payload["action"] == "discard"
|
||||
assert payload["reason"] == "mid_stream_failure"
|
||||
assert "stream_exec_id" in payload
|
||||
assert "lower API priority" in payload["message"]
|
||||
assert "未完成的输出已丢弃" in payload["message"]
|
||||
assert "lower API priority" not in payload["message"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -0,0 +1,141 @@
|
||||
"""Runaway click arguments must never reach the executor or endless retries."""
|
||||
|
||||
from copy import deepcopy
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
from langchain_core.messages import AIMessageChunk
|
||||
import pytest
|
||||
|
||||
from artemis.llm.reliability import FailureCategory, LLMExhaustedError, retry_policy_for
|
||||
from artemis.llm.router import ModelEndpoint
|
||||
from artemis.llm.stream_guard import (
|
||||
SMALL_ACTION_ARGUMENT_LIMIT,
|
||||
ToolArgumentGuard,
|
||||
ToolArgumentOverflow,
|
||||
)
|
||||
from artemis.services import llm
|
||||
|
||||
|
||||
def delta(arguments, *, name=None, index=0):
|
||||
return AIMessageChunk(
|
||||
content=[],
|
||||
tool_call_chunks=[{"name": name, "args": arguments, "index": index}],
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def isolate(monkeypatch, tmp_path):
|
||||
llm._ENDPOINT_BREAKER._states.clear()
|
||||
llm._NON_STREAMING_ENDPOINTS.clear()
|
||||
monkeypatch.setattr(llm, "PAUSE_FILE", tmp_path / "pause")
|
||||
monkeypatch.setattr(
|
||||
llm, "retry_policy_for", lambda _: SimpleNamespace(max_attempts=2, delay_for=lambda _: 0)
|
||||
)
|
||||
monkeypatch.setattr(llm, "_wait_for_resume", AsyncMock(return_value=False))
|
||||
monkeypatch.setattr(llm, "_CURRENT_DATA_ENGINE", None)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", ["save_note", "input_text", "report_task_status", "plugin_tool"])
|
||||
def test_free_text_tools_and_regular_text_are_not_limited(name):
|
||||
guard = ToolArgumentGuard()
|
||||
text = "报告内容" * SMALL_ACTION_ARGUMENT_LIMIT
|
||||
guard.observe(AIMessageChunk(content=text))
|
||||
guard.observe(delta(json.dumps({"text": text}, ensure_ascii=False), name=name))
|
||||
|
||||
|
||||
def test_split_and_interleaved_arguments_are_counted_per_call_without_storing_body():
|
||||
guard = ToolArgumentGuard()
|
||||
for index in (0, 1):
|
||||
guard.observe(delta('{"target_description":"', name="click", index=index))
|
||||
for index in (0, 1):
|
||||
guard.observe(delta("private" * 1200, index=index))
|
||||
with pytest.raises(ToolArgumentOverflow) as caught:
|
||||
guard.observe(delta("private" * 1200, index=1))
|
||||
assert caught.value.details["tool"] == "click"
|
||||
assert caught.value.details["argument_chars"] > SMALL_ACTION_ARGUMENT_LIMIT
|
||||
assert "private" not in str(caught.value)
|
||||
assert "private" not in repr(guard.calls)
|
||||
|
||||
|
||||
def test_name_arriving_after_arguments_still_triggers_guard():
|
||||
guard = ToolArgumentGuard()
|
||||
guard.observe(delta("x" * (SMALL_ACTION_ARGUMENT_LIMIT + 1)))
|
||||
with pytest.raises(ToolArgumentOverflow):
|
||||
guard.observe(delta("", name="click"))
|
||||
|
||||
|
||||
def test_invalid_output_retry_policy_is_bounded_and_honors_zero_retries(monkeypatch):
|
||||
monkeypatch.setattr(llm.settings, "OPENAI_MAX_RETRIES", 5)
|
||||
assert retry_policy_for(FailureCategory.INVALID_OUTPUT).max_attempts == 2
|
||||
monkeypatch.setattr(llm.settings, "OPENAI_MAX_RETRIES", 0)
|
||||
assert retry_policy_for(FailureCategory.INVALID_OUTPUT).max_attempts == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("recover", [False, True])
|
||||
@pytest.mark.parametrize("first_chunk_overflows", [False, True])
|
||||
async def test_gateway_closes_bad_stream_discards_partial_and_bounds_retry(
|
||||
monkeypatch, recover, first_chunk_overflows
|
||||
):
|
||||
records, resets, closed = [], [], []
|
||||
|
||||
class Engine:
|
||||
current_session_id = "test-session"
|
||||
current_step_id = None
|
||||
|
||||
def record_trace(self, **record):
|
||||
records.append(deepcopy(record))
|
||||
|
||||
def _publish(self, *_):
|
||||
pass
|
||||
|
||||
class Model:
|
||||
calls = 0
|
||||
|
||||
async def astream(self, *_args, **_kwargs):
|
||||
self.calls += 1
|
||||
try:
|
||||
if recover and self.calls == 2:
|
||||
yield delta('{"target":3}', name="click")
|
||||
return
|
||||
if not first_chunk_overflows:
|
||||
yield delta('{"target_description":"', name="click")
|
||||
yield delta(
|
||||
"private" * (SMALL_ACTION_ARGUMENT_LIMIT // 7 + 1),
|
||||
name="click" if first_chunk_overflows else None,
|
||||
)
|
||||
pytest.fail("Runaway stream was consumed after the safety bound")
|
||||
finally:
|
||||
closed.append(self.calls)
|
||||
|
||||
monkeypatch.setattr(llm, "_CURRENT_DATA_ENGINE", Engine())
|
||||
monkeypatch.setattr(llm, "_record_llm_event", lambda *a, **_: resets.append(a))
|
||||
model = Model()
|
||||
wrapper = llm.RobustChatModelWrapper(model, endpoint=ModelEndpoint(max_retries=5))
|
||||
if recover:
|
||||
response = await wrapper.complete([])
|
||||
assert response.tool_calls == [
|
||||
{"name": "click", "args": {"target": 3}, "id": None, "type": "tool_call"}
|
||||
]
|
||||
llm._wait_for_resume.assert_not_awaited()
|
||||
else:
|
||||
with pytest.raises(LLMExhaustedError) as caught:
|
||||
await wrapper.complete([])
|
||||
assert caught.value.failure.category is FailureCategory.INVALID_OUTPUT
|
||||
llm._wait_for_resume.assert_awaited_once()
|
||||
assert model.calls == 2
|
||||
assert closed == [1, 2]
|
||||
failure = next(
|
||||
r["payload"]
|
||||
for r in records
|
||||
if r["type"] == "llm_diagnostic" and r["payload"]["phase"] == "failed"
|
||||
)
|
||||
assert failure["stream_guard"]["tool"] == "click"
|
||||
assert failure["stream_guard"]["argument_chars"] > SMALL_ACTION_ARGUMENT_LIMIT
|
||||
assert failure["failure_phase"] == "generating_tool_arguments"
|
||||
assert failure["category"] == "invalid_output"
|
||||
assert failure["usage"] is None
|
||||
assert "private" not in json.dumps(records + resets, default=str)
|
||||
assert any(name == "llm_stream_reset" and p["action"] == "discard" for name, p in resets)
|
||||
@@ -45,7 +45,7 @@ def test_memory_defaults():
|
||||
assert cfg.memory.transcript.xml_scrub_depth == 1
|
||||
# Ready chunks replace their turns only once the
|
||||
# operator's measured context reaches budget*start_ratio.
|
||||
assert cfg.memory.chunking.min_steps == 3
|
||||
assert cfg.memory.chunking.min_steps == 6
|
||||
assert cfg.memory.transcript.image_scrub_depth_relaxed == 6
|
||||
assert cfg.memory.transcript.start_ratio == 0.35
|
||||
assert cfg.memory.transcript.soft_ratio == 0.7
|
||||
|
||||
@@ -19,9 +19,65 @@ from artemis.data_engine.models import ImageRecord, SessionMetadata, StepRecord
|
||||
from artemis.data_engine.storage import StorageManager
|
||||
from artemis.memory.transcript import TranscriptLedger
|
||||
from artemis.plugins.results import plugin_result_messages
|
||||
from artemis.plugins.models import PluginCallResult
|
||||
from artemis.plugins.runtime import PluginRuntime
|
||||
from artemis.tools.history.screenshot import load_step_screenshot
|
||||
|
||||
|
||||
def test_large_tree_deduplicates_without_mutating_raw_evidence():
|
||||
tree = "- Button abc\n" * 3000
|
||||
result = PluginCallResult(
|
||||
status="success",
|
||||
plugin_id="test",
|
||||
content=[{"type": "text", "text": "Screenshot\n" + tree}],
|
||||
structured_content={
|
||||
"tree_markdown": tree,
|
||||
"capture_id": "capture-current",
|
||||
"window_bounds": {"width": 2560},
|
||||
},
|
||||
)
|
||||
text = result.text()
|
||||
assert text.count(tree) == 1
|
||||
assert result.structured_content["tree_markdown"] == tree
|
||||
assert "capture-current" in text and "2560" in text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ["flash", "pro"])
|
||||
def test_verified_receipts_survive_observation_truncation_for_both_modes(mode):
|
||||
receipt = {
|
||||
"path": "artifacts/plugins/test/screen.png",
|
||||
"sha256": "a" * 64,
|
||||
"bytes": 123,
|
||||
"media_type": "image/png",
|
||||
}
|
||||
result = PluginCallResult(
|
||||
status="success",
|
||||
plugin_id="test",
|
||||
content=[{"type": "text", "text": "TREE " * 10000}],
|
||||
attachments=[receipt],
|
||||
)
|
||||
if mode == "flash":
|
||||
blocks = PluginRuntime.content_blocks(result)
|
||||
else:
|
||||
action = _action("call", attachments=[receipt])
|
||||
action["result"] = result.text()
|
||||
blocks = plugin_result_messages({"execution": [action]}, None)["call"][0].content
|
||||
text = str(blocks)
|
||||
assert receipt["path"] in text and receipt["sha256"] in text
|
||||
assert "verified file receipts" in text
|
||||
assert "观察文本已节选" in text
|
||||
assert len(text) < 14000
|
||||
|
||||
|
||||
def test_plugin_claim_does_not_become_runtime_receipt():
|
||||
result = PluginCallResult(
|
||||
status="success",
|
||||
plugin_id="test",
|
||||
structured_content={"attachments": [{"path": "pretend.png", "verified": True}]},
|
||||
)
|
||||
assert "verified file receipts" not in str(PluginRuntime.content_blocks(result))
|
||||
|
||||
|
||||
def _action(call_id, name="plug_test_get_app_state", **kwargs):
|
||||
return dict(
|
||||
action=name,
|
||||
|
||||
@@ -0,0 +1,257 @@
|
||||
"""Read-only diagnostics and destructive-action gates against disposable task data."""
|
||||
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
from uuid import uuid4
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
import pytest
|
||||
|
||||
from apps.admin_console.database.repositories.session_repository import SessionRepository
|
||||
from apps.admin_console.routers import sessions
|
||||
from apps.admin_console.services import report_export, task_deletion, task_inspection
|
||||
from apps.admin_console.services.automation_store import AutomationStore
|
||||
from apps.admin_console.services.request_diagnostics import request_diagnostics
|
||||
from artemis.data_engine.storage import StorageManager
|
||||
from artemis.runtime import trace_store
|
||||
from mcp_server.tools import automation, exploration, inspect_trace, task_manager
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def task(tmp_path, monkeypatch):
|
||||
sid = str(uuid4())
|
||||
db = tmp_path / "data_engine.db"
|
||||
StorageManager(db, tmp_path)
|
||||
store = AutomationStore(db)
|
||||
repo = SessionRepository(db)
|
||||
with store.connection() as connection:
|
||||
connection.execute(
|
||||
"INSERT INTO sessions (session_id, initial_goal, start_time, status, device_info) "
|
||||
"VALUES (?, 'observe', 1, 'completed', '{}')",
|
||||
(sid,),
|
||||
)
|
||||
connection.commit()
|
||||
monkeypatch.setattr(task_inspection, "session_repo", repo)
|
||||
monkeypatch.setattr(task_inspection, "store", store)
|
||||
monkeypatch.setattr(task_inspection.evidence_service, "task_root", lambda _: tmp_path / sid)
|
||||
monkeypatch.setattr(trace_store, "TRACES_DIR", str(tmp_path))
|
||||
monkeypatch.setattr(report_export, "store", store)
|
||||
monkeypatch.setattr(report_export, "TRACES_PATH", tmp_path)
|
||||
(tmp_path / sid).mkdir()
|
||||
return sid, db, tmp_path, store
|
||||
|
||||
|
||||
def add_trace(store, sid, kind, name, payload, timestamp=1):
|
||||
with store.connection() as connection:
|
||||
connection.execute(
|
||||
"INSERT INTO traces (trace_id,session_id,type,name,payload,timestamp) VALUES (?,?,?,?,?,?)",
|
||||
(str(uuid4()), sid, kind, name, json.dumps(payload), timestamp),
|
||||
)
|
||||
connection.commit()
|
||||
|
||||
|
||||
def test_usage_uses_input_denominator_and_does_not_invent_missing_model(task):
|
||||
sid, _, _, store = task
|
||||
assert task_inspection.inspect_task(sid, "usage")["cache_hit_rate"] is None
|
||||
add_trace(
|
||||
store,
|
||||
sid,
|
||||
"llm_call",
|
||||
"llm_usage",
|
||||
{
|
||||
"node": "operator",
|
||||
"model": "model-a",
|
||||
"prompt_tokens": 1000,
|
||||
"cached_tokens": 900,
|
||||
"completion_tokens": 500,
|
||||
},
|
||||
)
|
||||
add_trace(
|
||||
store,
|
||||
sid,
|
||||
"llm_call",
|
||||
"llm_usage",
|
||||
{
|
||||
"node": "checker",
|
||||
"prompt_tokens": 1000,
|
||||
"cached_tokens": 0,
|
||||
"completion_tokens": 500,
|
||||
},
|
||||
)
|
||||
result = task_inspection.inspect_task(sid, "usage")
|
||||
assert result["cache_hit_rate"] == 0.45
|
||||
assert result["total_tokens"] == 3000
|
||||
assert result["by_node_model"][1]["model"] == "unknown"
|
||||
|
||||
|
||||
def test_log_tail_is_bounded_paginated_and_redacted(task, monkeypatch):
|
||||
sid, _, root, _ = task
|
||||
monkeypatch.setenv("TEST_API_KEY", "test-only-sensitive-value")
|
||||
lines = ["x" * 200 for _ in range(1000)] + [
|
||||
"old",
|
||||
"test-only-sensitive-value",
|
||||
"Bearer abcdef",
|
||||
"password=hush",
|
||||
]
|
||||
(root / sid / "stderr.log").write_text("\n".join(lines), "utf-8")
|
||||
result = task_inspection.inspect_task(sid, "logs", limit=2)
|
||||
error = result["logs"][1]
|
||||
assert error["older_bytes_omitted"] > 0
|
||||
assert error["next_offset"] == 2
|
||||
assert error["lines"] == ["Bearer [REDACTED]", "password=[REDACTED]"]
|
||||
older = task_inspection.inspect_task(sid, "logs", offset=2, limit=2)
|
||||
assert older["logs"][1]["lines"] == ["old", "[REDACTED]"]
|
||||
assert result["logs"][0]["available"] is False
|
||||
|
||||
|
||||
def test_report_pagination_preserves_total_and_caps_nested_content(task, monkeypatch):
|
||||
sid, _, _, _ = task
|
||||
monkeypatch.setattr(
|
||||
report_export,
|
||||
"session_report",
|
||||
lambda _: {
|
||||
"steps": [{"number": i} for i in range(35)],
|
||||
"content": "x" * 100000,
|
||||
"metadata": {"api_key": "hidden"},
|
||||
"nested": [[{"value": "x" * 100} for _ in range(1000)] for _ in range(1000)],
|
||||
},
|
||||
)
|
||||
result = task_inspection.inspect_task(sid, "report", offset=20, limit=10)
|
||||
assert result["steps"]["items"] == [{"number": i} for i in range(20, 30)]
|
||||
assert result["steps"]["total"] == 35
|
||||
assert result["steps"]["next_offset"] == 30
|
||||
assert result["metadata"]["api_key"] == "[REDACTED]"
|
||||
assert result["content_truncated"] is True
|
||||
assert len(json.dumps(result, ensure_ascii=False)) < 150000
|
||||
|
||||
|
||||
def test_request_pagination_reports_latest_phase_only(task):
|
||||
sid, db, _, store = task
|
||||
with store.connection() as connection:
|
||||
connection.execute("UPDATE sessions SET status='running' WHERE session_id=?", (sid,))
|
||||
connection.commit()
|
||||
add_trace(
|
||||
store, sid, "llm_diagnostic", "request", {"request_id": "r", "phase": "waiting_response"}, 1
|
||||
)
|
||||
add_trace(store, sid, "llm_diagnostic", "request", {"request_id": "r", "phase": "completed"}, 2)
|
||||
assert request_diagnostics(sid, db_path=db, limit=1)["next_offset"] == 1
|
||||
older = request_diagnostics(sid, db_path=db, limit=1, offset=1)
|
||||
assert older["requests"][0]["active"] is False
|
||||
assert older["next_offset"] is None
|
||||
|
||||
|
||||
def test_status_and_report_use_same_human_review_without_losing_ai_result(task):
|
||||
sid, db, root, store = task
|
||||
(root / sid / "run_outcome.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"test_result": {
|
||||
"status": "failed",
|
||||
"reason": "AI found mismatch",
|
||||
"evidence": ["step 1: actual differs from expected"],
|
||||
},
|
||||
}
|
||||
),
|
||||
"utf-8",
|
||||
)
|
||||
store.put(
|
||||
"case_review",
|
||||
sid,
|
||||
{"reviews": [{"status": "passed", "reason": "用户确认", "reviewed_at": 10}]},
|
||||
)
|
||||
result = task_manager.mobile_manage_task("status", sid)["test_result"]
|
||||
assert result == report_export.session_report(sid, db_path=db)["test_result"]
|
||||
assert result["source"] == "human"
|
||||
assert result["ai_result"]["status"] == "failed"
|
||||
|
||||
|
||||
def test_inspection_not_found_and_pagination_validation(task):
|
||||
app = FastAPI()
|
||||
app.include_router(sessions.router)
|
||||
client = TestClient(app)
|
||||
assert client.get(f"/api/sessions/{uuid4()}/inspection").status_code == 404
|
||||
assert client.get(f"/api/sessions/{task[0]}/inspection?limit=101").status_code == 422
|
||||
|
||||
|
||||
def test_active_normal_task_delete_is_rejected_without_touching_history(task, monkeypatch):
|
||||
sid, _, root, store = task
|
||||
monkeypatch.setattr(task_deletion, "session_repo", task_inspection.session_repo)
|
||||
monkeypatch.setattr(task_deletion, "store", store)
|
||||
monkeypatch.setattr(task_deletion, "_check_runs", lambda _: None)
|
||||
store.save_job({"session_id": sid, "status": "running", "position": 1})
|
||||
app = FastAPI()
|
||||
app.include_router(sessions.router)
|
||||
client = TestClient(app)
|
||||
assert client.get(f"/api/sessions/{sid}/deletion-preview").status_code == 409
|
||||
assert client.post(f"/api/sessions/{sid}/delete").status_code == 409
|
||||
assert (root / sid).is_dir()
|
||||
assert store.job(sid)["status"] == "running"
|
||||
|
||||
|
||||
def test_mcp_review_and_delete_require_user_intent_and_share_http_backend(monkeypatch):
|
||||
request = AsyncMock(return_value={"test_result": {"source": "human"}})
|
||||
monkeypatch.setattr(automation, "_request", request)
|
||||
for action in ("review", "delete"):
|
||||
with pytest.raises(ValueError, match="用户"):
|
||||
task_manager.mobile_manage_task(action, "task")
|
||||
request.assert_not_awaited()
|
||||
result = task_manager.mobile_manage_task(
|
||||
"review", "task", review_status="passed", review_reason="确认", user_confirmed=True
|
||||
)
|
||||
assert result["test_result"]["source"] == "human"
|
||||
request.assert_awaited_once_with(
|
||||
"PUT", "/api/automation/reports/task/review", json={"status": "passed", "reason": "确认"}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mcp_read_inspection_and_exploration_delete_routes(monkeypatch):
|
||||
request = AsyncMock(return_value={"ok": True})
|
||||
monkeypatch.setattr(automation, "_request", request)
|
||||
monkeypatch.setattr(exploration, "_request", request)
|
||||
await inspect_trace.mobile_inspect_trace("view_requests", "task", offset=20, limit=10)
|
||||
request.assert_awaited_with(
|
||||
"GET",
|
||||
"/api/sessions/task/inspection",
|
||||
params={"view": "requests", "offset": 20, "limit": 10},
|
||||
)
|
||||
with pytest.raises(ValueError, match="用户"):
|
||||
await exploration.mobile_exploration("delete", exploration_id="e")
|
||||
await exploration.mobile_exploration("delete", exploration_id="e", user_confirmed=True)
|
||||
request.assert_awaited_with("DELETE", "/api/explorations/e")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_plugin_probe_only_targets_installed_plugin(monkeypatch):
|
||||
async def request(method, path, **kwargs):
|
||||
return {"plugins": [{"manifest": {"id": "demo"}}]} if path == "/api/plugins" else {}
|
||||
|
||||
mocked = AsyncMock(side_effect=request)
|
||||
monkeypatch.setattr(automation, "_request", mocked)
|
||||
result = await automation.mobile_workspace(
|
||||
include_devices=False, plugin_id="demo", probe_plugin=True, include_development=True
|
||||
)
|
||||
assert result["plugins"][0]["manifest"]["id"] == "demo"
|
||||
assert result["development"]["plugin_schema_version"] == 2
|
||||
assert [call.args for call in mocked.call_args_list if call.args[0] == "POST"] == [
|
||||
("POST", "/api/plugins/demo/diagnose")
|
||||
]
|
||||
|
||||
|
||||
def test_exploration_deletion_preview_keeps_validation_runs(monkeypatch):
|
||||
repository = SimpleNamespace(
|
||||
get=lambda _: {},
|
||||
runs=lambda _: [
|
||||
{"id": "explore-run", "kind": "exploration"},
|
||||
{"id": "rerun", "kind": "validation"},
|
||||
],
|
||||
)
|
||||
monkeypatch.setattr(task_deletion, "explorations", repository)
|
||||
check = Mock()
|
||||
monkeypatch.setattr(task_deletion, "_check_runs", check)
|
||||
result = task_deletion.exploration_deletion_preview("e")
|
||||
assert result["session_ids"] == ["explore-run"]
|
||||
assert result["preserved_session_ids"] == ["rerun"]
|
||||
check.assert_called_once()
|
||||
@@ -17,11 +17,34 @@ from artemis.utils.notes import (
|
||||
list_notes_info,
|
||||
read_note_content,
|
||||
save_note_content,
|
||||
save_verified_note,
|
||||
update_note_content,
|
||||
)
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.parametrize("content", ["# 报告\n结果\n", "line\r\nsecond", ""])
|
||||
def test_verified_note_receipt_matches_actual_file_bytes(tmp_path, content):
|
||||
import hashlib
|
||||
|
||||
receipt = save_verified_note(tmp_path, "report.md", content)
|
||||
data = (tmp_path / receipt["path"]).read_bytes()
|
||||
assert receipt["path"] == "notes/report.md"
|
||||
assert receipt["verified"] is True and receipt["bytes"] == len(data)
|
||||
assert receipt["sha256"] == hashlib.sha256(data).hexdigest()
|
||||
|
||||
|
||||
def test_verified_note_does_not_confirm_incomplete_write(tmp_path, monkeypatch):
|
||||
from artemis.utils import notes
|
||||
|
||||
def partial_write(base_dir, key, content):
|
||||
save_note_content(base_dir, key, content[:1])
|
||||
|
||||
monkeypatch.setattr(notes, "save_note_content", partial_write)
|
||||
with pytest.raises(OSError, match="读回不一致"):
|
||||
save_verified_note(tmp_path, "report", "完整报告")
|
||||
|
||||
|
||||
def test_list_notes_info(tmp_path):
|
||||
base_dir = tmp_path
|
||||
|
||||
|
||||
Reference in new issue
Block a user