feat: add official Codex skill and harden task execution

Install and update artemis-use with task diagnostics and plugin guidance. Preserve cross-device file receipts, review canonical reports, guard stage transitions and reduce small history summaries. Improve exploration lifecycle, request recovery and browser live connections.
This commit is contained in:
chenmingxuan committed 2026-10-18 14:18:37 +08:00
1 parent 2328a4df0c
commit e1ea099592
128 files changed
+4944 -700

No files matched your search

+6
View File
@@ -84,3 +84,9 @@ def pytest_collection_modifyitems(items: list[pytest.Item]) -> None:
item.add_marker(pytest.mark.integration)
if "e2e" in parts:
item.add_marker(pytest.mark.e2e)
@pytest.fixture(autouse=True)
def no_host_codex_startup_sync(monkeypatch):
"""Starting an isolated test app must never modify the real Codex profile."""
monkeypatch.setenv("ARTEMIS_SKIP_CODEX_SYNC", "1")
+54 -2
View File
@@ -673,7 +673,7 @@ def test_format_history_uses_session_offsets_and_skips_empty_steps():
assert "'Login'" in lines[1]
async def _final_check_human_text(ctx) -> str:
async def _final_check_human_text(ctx, **kwargs) -> str:
report = CheckReport(verdicts=[])
structured = MagicMock()
structured.ainvoke = AsyncMock(return_value=report)
@@ -691,7 +691,9 @@ async def _final_check_human_text(ctx) -> str:
new=AsyncMock(return_value=(None, "elements")),
),
):
await run_final_check(ctx, goal="the goal", plan_text="- [x] G", ledger=[], check_items=[])
await run_final_check(
ctx, goal="the goal", plan_text="- [x] G", ledger=[], check_items=[], **kwargs
)
messages = acomplete_mock.await_args.args[1]
human = messages[-1]
@@ -730,6 +732,56 @@ async def test_final_review_reports_no_guidance_explicitly():
assert "# User guidance received during the run\n(none)\n" in text
@pytest.mark.asyncio
async def test_final_review_receives_verified_file_receipts_without_history_tool_call():
ctx = _mock_ctx()
ctx.data_engine.get_agent_friendly_steps.return_value = [
{
"step_number": 44,
"summary": "已写入报告",
"action_taken": {"action": "android_file"},
"last_execution_result": {
"result": '{"ok":true,"verified":true,"device_path":"/sdcard/report.md","bytes":8022}'
},
}
]
text = await _final_check_human_text(ctx, execution_report="报告保存情况未知")
assert "Recorded File Receipts (tool evidence, not report claims)" in text
assert '"step": 44' in text and '"verified": true' in text
assert "/sdcard/report.md" in text
@pytest.mark.parametrize("entry", ["checkpoint", "final"])
def test_visual_acceptance_requires_counter_evidence_without_exempting_real_defects(entry):
text = assemble_checker_prompt_segments(entry, [], probe_tool_registered=False)
assert "compare recorded pre/post screenshots and nearby steps" in text
assert "user permits this behavior" in text
assert "fixed navigation is not a blanket exemption" in text
assert "explicit visual standard" in text
assert "inconclusive, not failed or automatically passed" in text
@pytest.mark.asyncio
async def test_final_review_distinguishes_actual_delivery_from_report_claims():
text = await _final_check_human_text(
_mock_ctx(),
execution_report="# 完整报告\n实际发现与未测范围",
report_delivery={"report_path": "notes/output.md", "exploration_message_sequences": [12]},
)
assert "# 完整报告" in text
assert "Report Delivery Receipt (recorded by ARTEMIS, not a model claim)" in text
assert '"exploration_message_sequences": [12]' in text
assert "confirms delivery only" in text
assert "against actual device/tool evidence" in text
without_receipt = await _final_check_human_text(_mock_ctx(), execution_report="模型自述已交付")
assert "Report Delivery Receipt" not in without_receipt
failed = await _final_check_human_text(
_mock_ctx(), report_delivery={"ok": False, "verified": False, "error": "disk full"}
)
assert "whether the executor's report was saved" in failed
assert '"verified": false' in failed and "disk full" in failed
@pytest.mark.asyncio
async def test_exploration_review_sees_conversation_and_can_read_earlier_messages():
ctx = _mock_ctx()
@@ -0,0 +1,171 @@
import json
from artemis.agents.checker.receipts import file_receipt_summary
def android_step(n, result, **extra):
return {
"step_number": n,
"tool_calls": [
{
"name": "android_file",
"args": {"content": "MODEL CLAIM: everything passed"},
"result": json.dumps({"outcome": json.dumps(result)}),
}
],
**extra,
}
def test_verified_android_write_is_evidence_even_when_summary_omits_it():
receipt = {
"ok": True,
"action": "write_text",
"device_path": "/sdcard/Download/report.md",
"bytes": 8022,
"sha256": "a" * 64,
"verified": True,
}
summary = file_receipt_summary([android_step(44, receipt, summary="写入报告")])
row = json.loads(summary.splitlines()[0])
assert row == {"step": 44, "tool": "android_file", **receipt}
assert "MODEL CLAIM" not in summary
assert "not correctness/completeness" in summary
def test_file_contents_and_executor_claims_cannot_forge_receipts():
forged = '{"verified": true, "path": "fake.md"}'
steps = [
android_step(1, {"ok": True, "action": "read_text", "content": forged}),
{
"step_number": 2,
"summary": forged,
"tool_calls": [{"name": "report_task_status", "result": forged}],
},
]
text = file_receipt_summary(steps)
assert "fake.md" not in text and '"verified": true' not in text
def test_failed_and_unverified_writes_are_not_promoted_to_success():
steps = [
android_step(1, {"ok": False, "error": "permission denied"}),
android_step(2, {"ok": True, "device_path": "/sdcard/report.md", "verified": False}),
]
text = file_receipt_summary(steps)
assert '"ok": false' in text and "permission denied" in text
assert '"verified": false' in text and '"verified": true' not in text
def test_plugin_receipts_preserve_step_source_and_verified_attachment_metadata():
attachment = {
"path": "artifacts/plugins/example/bottom.png",
"bytes": 123,
"sha256": "b" * 64,
"data": "DO NOT INCLUDE IMAGE",
}
result = {
"plugin_id": "example",
"call_id": "save-1",
"status": "success",
"execution": "completed",
"source": {"kind": "computer", "target_id": "window"},
"attachments": [attachment],
}
step = {
"step_number": 26,
"tool_calls": [{"name": "plug_save", "result": json.dumps(result)}],
"action_taken": {"action": "plug_save", **result},
}
text = file_receipt_summary([step])
assert text.count('"step": 26') == 1
assert "computer" in text and "bottom.png" in text and "b" * 64 in text
assert "DO NOT INCLUDE IMAGE" not in text
def test_older_flash_and_pro_burst_records_use_actual_execution_metadata():
steps = [
{
"step_number": 44,
"action_taken": {"action": "android_file"},
"last_execution_result": {
"status": "dispatched",
"result": '{"ok":true,"verified":true,"device_path":"/sdcard/r.md"}',
},
},
{
"step_number": 45,
"last_execution_result": {
"execution": [
{
"action": "plug_save",
"plugin_id": "example",
"plugin_status": "success",
"attachments": [{"path": "artifacts/a.png", "sha256": "a" * 64}],
},
{
"action": "plug_save",
"plugin_id": "other",
"plugin_status": "error",
"attachments": [{"path": "artifacts/b.png", "sha256": "b" * 64}],
},
]
},
},
]
text = file_receipt_summary(steps)
assert "/sdcard/r.md" in text and '"verified": true' in text
assert "artifacts/a.png" in text and "artifacts/b.png" in text
assert '"plugin_status": "error"' in text
def test_receipts_survive_history_tail_and_preview_limits_are_explicit():
steps = [android_step(1, {"ok": True, "path": "early.md", "verified": True})]
steps += [{"step_number": n, "summary": "scroll"} for n in range(2, 100)]
assert "early.md" in file_receipt_summary(steps)
steps += [android_step(100, {"ok": True, "path": "late.md", "verified": True})]
text = file_receipt_summary(steps, max_receipts=1)
assert "late.md" in text and "early.md" not in text
assert "1 receipt(s) omitted" in text
assert "not proof of a missing file" in text
tiny = file_receipt_summary(steps, max_chars=1)
assert "2 receipt(s) omitted" in tiny
def test_malformed_or_absent_receipt_is_not_silently_verified():
steps = [
{
"step_number": 1,
"tool_calls": [
{"name": "android_file", "result": "安卓文件操作失败:permission denied"},
{"name": "plug_save", "result": "invalid JSON"},
],
}
]
assert "permission denied" in file_receipt_summary(steps)
assert '"verified": true' not in file_receipt_summary(steps)
assert "unavailable" in file_receipt_summary("read failed")
def test_flash_saved_note_and_legacy_pro_note_preserve_actual_receipt_semantics():
steps = [
{
"step_number": 30,
"tool_calls": [
{
"name": "save_note",
"args": {"content": "DO NOT COPY THE REPORT BODY"},
"result": {"outcome": '{"ok":true,"path":"notes/report.md","verified":true}'},
},
{"name": "save_note", "result": "Note 'legacy' saved successfully."},
{"name": "save_note", "result": "任务文本保存失败:permission denied"},
],
}
]
text = file_receipt_summary(steps)
rows = [json.loads(line) for line in text.splitlines()[:3]]
assert rows[0]["verified"] is True and rows[0]["path"] == "notes/report.md"
assert rows[1]["recorded_result"] == "Note 'legacy' saved successfully."
assert "error" not in rows[1] and "verified" not in rows[1]
assert "permission denied" in rows[2]["recorded_result"]
assert "verified" not in rows[2] and "DO NOT COPY" not in text
@@ -87,6 +87,58 @@ async def test_complete_checker_json_does_not_request_another_verdict(status):
llm.with_structured_output.assert_not_called()
@pytest.mark.asyncio
@pytest.mark.parametrize("status", ["passed", "failed", "inconclusive"])
async def test_complete_verdict_alias_and_evidence_list_are_not_sent_for_reformatting(status):
payload = {
"verdicts": [
{
"kind": "assert",
"item_text": "检查本轮页面",
"verdict": status,
"evidence": ["步骤 2 的实际页面", "步骤 3 的对照截图"],
}
],
"test_result": _result(status),
}
llm, structured = _llm(CheckReport())
report = await _structured_report(
llm,
[AIMessage(content=json.dumps(payload))],
[SimpleNamespace(kind="assert", text="检查本轮页面")],
)
assert report.verdicts[0].status == status
assert report.verdicts[0].evidence == "步骤 2 的实际页面\n步骤 3 的对照截图"
assert report.test_result.model_dump() == payload["test_result"]
structured.ainvoke.assert_not_awaited()
llm.with_structured_output.assert_not_called()
@pytest.mark.parametrize(
"fields",
[
{"status": "failed", "verdict": "passed", "evidence": ["图 1"]},
{"verdict": "unknown", "evidence": ["图 1"]},
{"verdict": "passed", "evidence": [{"text": "图 1"}]},
{"verdict": "passed", "evidence": [1]},
],
)
def test_conflicting_or_invalid_verdict_shapes_are_not_silently_accepted(fields):
with pytest.raises(ValidationError):
CheckReport.model_validate(
{"verdicts": [{"kind": "assert", "item_text": "页面", **fields}]}
)
def test_empty_evidence_list_does_not_turn_an_unsupported_failure_into_evidence():
from artemis.agents.checker.checker import _normalize_report
report = CheckReport.model_validate(
{"verdicts": [{"kind": "assert", "item_text": "页面", "verdict": "failed", "evidence": []}]}
)
assert _normalize_report(report, []).verdicts[0].status == "inconclusive"
def test_nested_json_string_result_is_decoded_and_still_validated():
result = _result()
assert (
@@ -64,6 +64,20 @@ def test_phone_and_unchanged_pc_captures_are_both_sent_with_origin(tmp_path):
assert "Step 27" in text and "Step 28" in text
assert "identical to pre-capture" in text
assert "not proof of success" in text
assert "saved screenshot file=" in text
assert "Distinct capture events may share one file" in text
def test_unchanged_android_boundary_capture_is_saved_evidence_not_missing_export(tmp_path):
footer = image(tmp_path, b"same-footer")
blocks = final_evidence(
engine(tmp_path, [step(26, post=footer), step(28, pre=footer, post=footer)])
)
text = str(blocks)
assert "Step 26" in text and "Step 28" in text
assert str(tmp_path / f"{footer}.jpg") in "\n".join(b.get("text", "") for b in blocks)
assert "automatic trace screenshot is recorded evidence" in text
assert sum(b["type"] == "image_url" for b in blocks) == 1
def test_native_resolution_change_does_not_hide_previous_same_window_capture(tmp_path):
@@ -111,3 +125,13 @@ def test_later_call_without_observation_marks_previous_picture_historical(tmp_pa
assert "Step 2" in str(blocks) and "no post-action image" in str(blocks)
assert "Earlier captures do not prove" in str(blocks)
assert sum(x["type"] == "image_url" for x in blocks) == 1
def test_file_receipt_is_linked_to_its_resource_image_without_an_extra_image(tmp_path):
name = image(tmp_path, b"pc")
action = pc(name)
action["attachments"] = [{"path": "artifacts/plugins/cua-driver/pc.png", "sha256": name}]
blocks = final_evidence(engine(tmp_path, [step(35, action=action)]))
assert "runtime-verified saved attachments" in str(blocks)
assert "artifacts/plugins/cua-driver/pc.png" in str(blocks)
assert sum(x["type"] == "image_url" for x in blocks) == 1
+181 -1
View File
@@ -6,11 +6,64 @@ from langchain_core.messages import AIMessage
import pytest
from artemis.agents.checker import flash
from artemis.agents.checker.checker import CheckReport
from artemis.agents.checker.checker import CheckReport, ReportCorrection
from artemis.context import ArtemisContext, ExecutionSetup
from artemis.runtime.case_result import CaseTestResult
@pytest.mark.asyncio
async def test_checker_corrects_report_with_evidence_without_changing_product_failure(
tmp_path, monkeypatch
):
from artemis.utils.notes import save_verified_note
body = "# 双端报告\nPC 为浅色;PC 截图没有保存;安卓导航遮挡。"
save_verified_note(tmp_path, "report", body)
corrections = [
ReportCorrection(
original="PC 为浅色", replacement="PC 为深色", evidence="步骤 35 电脑截图"
),
ReportCorrection(
original="PC 截图没有保存",
replacement="PC 截图已保存",
evidence="步骤 35 已验证 path/hash 回执",
),
]
check = AsyncMock(
return_value=CheckReport(
test_result=CaseTestResult(
status="failed", reason="安卓导航遮挡正文", evidence=["步骤 40"]
),
report_corrections=corrections,
)
)
monkeypatch.setattr(flash, "run_final_check", check)
execution = {"status": "completed", "explanation": body, "report_note_key": "report"}
data = await flash.settle_flash_outcome(context(tmp_path), "双端视觉检查", execution)
assert data["test_result"]["status"] == "failed"
assert data["report_review"]["applied"] is True
assert execution["explanation"] == (tmp_path / "notes/output.md").read_text(encoding="utf-8")
assert execution["explanation"] == (tmp_path / "notes/report.md").read_text(encoding="utf-8")
assert "PC 为深色" in execution["explanation"]
assert (tmp_path / "notes/output_before_review.md").read_text(encoding="utf-8") == body
stored = json.loads((tmp_path / "run_outcome.json").read_text(encoding="utf-8"))
assert stored["report_review"]["corrections"][0]["evidence"] == "步骤 35 电脑截图"
def test_report_correction_requires_exact_unique_quote(tmp_path):
from artemis.agents.checker.report_review import apply_report_review
execution = {"explanation": "same same"}
result = apply_report_review(
tmp_path,
execution,
[ReportCorrection(original="same", replacement="changed", evidence="步骤 1")],
)
assert result["applied"] is False
assert execution["explanation"] == "same same"
assert not (tmp_path / "notes/output.md").exists()
def context(tmp_path, **setup):
ctx = MagicMock(spec=ArtemisContext)
ctx.execution_setup = ExecutionSetup(**setup)
@@ -34,6 +87,10 @@ async def test_flash_reuses_checker_with_original_prompt_and_final_report(
async def review_after_capture_stops(*args, **kwargs):
ctx.on_execution_review.assert_awaited_once()
receipt = kwargs["report_delivery"]
assert receipt["verified"] is True
assert receipt["report_path"] == "notes/output.md"
assert (tmp_path / receipt["report_path"]).read_text(encoding="utf-8") == "操作已执行"
return CheckReport(test_result=result)
check.side_effect = review_after_capture_stops
@@ -80,6 +137,38 @@ async def test_disabled_acceptance_does_not_call_a_model_or_claim_pass(
assert data["test_result"]["status"] == "inconclusive"
@pytest.mark.asyncio
async def test_report_is_saved_without_checker_or_another_model(tmp_path, monkeypatch):
check = AsyncMock()
monkeypatch.setattr(flash, "run_final_check", check)
text = "# 双端报告\n安卓与电脑证据分开列出。\n## 未测\n没有验证播放。"
await flash.settle_flash_outcome(
context(tmp_path, disable_checker=True),
"报告",
{"status": "completed", "explanation": text},
)
assert (tmp_path / "notes/output.md").read_text(encoding="utf-8") == text
check.assert_not_awaited()
@pytest.mark.asyncio
async def test_report_write_failure_is_visible_to_checker_not_claimed_saved(tmp_path, monkeypatch):
(tmp_path / "notes").write_text("not a directory", encoding="utf-8")
check = AsyncMock(
return_value=CheckReport(
test_result=CaseTestResult(status="blocked", reason="报告无法保存")
)
)
monkeypatch.setattr(flash, "run_final_check", check)
outcome = await flash.settle_flash_outcome(
context(tmp_path), "报告", {"status": "completed", "explanation": "实际报告正文"}
)
receipt = check.call_args.kwargs["report_delivery"]
assert receipt["ok"] is False and receipt["verified"] is False
assert "report_path" not in receipt
assert outcome["test_result"]["status"] == "blocked"
@pytest.mark.asyncio
async def test_execution_failure_is_blocked_without_another_model_retry(tmp_path, monkeypatch):
check = AsyncMock()
@@ -156,3 +245,94 @@ async def test_complete_checker_answer_is_persisted_without_reformatting(tmp_pat
assert outcome["test_result"] == result.model_dump()
saved = json.loads((tmp_path / "run_outcome.json").read_text(encoding="utf-8"))
assert saved["test_result"] == result.model_dump()
@pytest.mark.asyncio
@pytest.mark.parametrize("execution_target", ["android", "computer"])
async def test_flash_text_delivery_reaches_checker_through_real_trace(
tmp_path, monkeypatch, execution_target
):
from artemis.agents.checker import checker
from artemis.agents.flash.runner import FlashRunner
from artemis.data_engine import engine as engine_module
from artemis.data_engine.engine import DataEngine
from artemis.graph.state import State
from artemis.memory.transcript import TranscriptLedger
from artemis.mcp.action_executor import McpActionExecutor
monkeypatch.setattr(engine_module, "PAUSE_FILE", tmp_path / "pause")
monkeypatch.setattr(engine_module, "_CURRENT_DATA_ENGINE", None)
ctx = context(tmp_path, traces_path=str(tmp_path))
ctx.device = None
ctx.exploration_id = None
ctx.on_execution_review = None
ctx.execution_target = execution_target
ctx.task_control = None
engine = DataEngine(ctx)
ctx.data_engine = engine
engine.start_session("保存交付文件并验收")
actuator = MagicMock()
actuator.extensions.return_value = []
executor = McpActionExecutor(ctx, actuator=actuator, agent_name="flash")
report_body = "# 双端报告\n实际发现及未测范围。"
verdict = CaseTestResult(status="inconclusive", reason="交付已保存,业务证据不足")
async def inspect_request(llm, messages, **kwargs):
text = "\n".join(block.get("text", "") for block in messages[-1].content)
assert '"tool": "save_note"' in text
assert '"path": "notes/findings.md"' in text and '"verified": true' in text
assert '"report_path": "notes/output.md"' in text
assert (engine.base_dir / "notes/output.md").read_text(encoding="utf-8") == report_body
assert (engine.base_dir / "notes/findings.md").read_text(encoding="utf-8") == "业务观察记录"
return AIMessage(content=json.dumps({"test_result": verdict.model_dump()}))
model = AsyncMock(side_effect=inspect_request)
monkeypatch.setattr(checker, "get_llm", lambda **_: MagicMock())
monkeypatch.setattr(checker, "acomplete", model)
monkeypatch.setattr(checker, "_capture_final_screen", AsyncMock(return_value=(None, "")))
monkeypatch.setattr("artemis.controllers.unified_controller.get_driver", lambda *_: MagicMock())
runner = FlashRunner(ctx, goal="核实双端布局并保存报告", max_turns=0)
runner.executor = executor
runner.summarizer = None
runner._init_llm = MagicMock(return_value=MagicMock())
runner._prepare_conversation = AsyncMock(return_value=(TranscriptLedger(), None, ""))
runner._refresh_observation = AsyncMock(side_effect=AssertionError("note must not observe"))
runner._android_observation_stale = True
runner._invoke_model = AsyncMock(
side_effect=[
AIMessage(
content="保存已观察结果。",
tool_calls=[
{
"name": "save_note",
"args": {"key": "findings", "content": "业务观察记录"},
"id": "save-1",
}
],
),
AIMessage(
content="提交报告。",
tool_calls=[
{
"name": "report_task_status",
"args": {"status": "completed", "explanation": report_body},
"id": "done",
}
],
),
]
)
try:
execution = await runner.run(State(initial_goal=runner.goal))
await engine.shutdown()
steps = engine.get_agent_friendly_steps()
assert steps[0]["action_taken"]["action"] == "save_note"
assert steps[0]["tool_calls"][0]["name"] == "save_note"
assert not steps[0].get("pre_image_name") and not steps[0].get("post_image_name")
runner._refresh_observation.assert_not_awaited()
outcome = await flash.settle_flash_outcome(ctx, runner.goal, execution)
model.assert_awaited_once()
assert outcome["test_result"] == verdict.model_dump()
actuator.execute.assert_not_called()
finally:
await engine.shutdown()
@@ -22,6 +22,9 @@ def cross_target(monkeypatch):
bindings={PC: ("pc", None)},
tool_names={PC},
previous_image=lambda *_: b"pc-before",
# This fixture only returns a PC observation; stage-write guards have
# dedicated coverage in test_flash_progress.py.
is_read_only=lambda _: True,
)
monkeypatch.setattr("artemis.plugins.runtime.get_plugin_runtime", lambda _: plugins)
runner = flash.FlashRunner.__new__(flash.FlashRunner)
+138
View File
@@ -0,0 +1,138 @@
from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock
import pytest
from pydantic import ValidationError
from artemis.agents.flash.progress import TaskProgress
from artemis.agents.flash.runner import FlashRunner, _TurnRecord
def plan(phone="active", pc="pending"):
return {
"reason": "按用户要求先手机页脚,再电脑独立检查",
"stages": [
{
"id": "phone",
"scope": "Android 主体、页脚及最后滚动",
"resources": ["android"],
"status": phone,
"evidence": "步骤 12 页脚及 13 滚动截图" if phone in {"done", "blocked"} else "",
},
{
"id": "pc",
"scope": "电脑检查",
"resources": ["cua-driver"],
"status": pc,
"evidence": "步骤 20 电脑截图" if pc == "done" else "",
},
],
}
def test_ordered_stages_require_evidence_and_keep_outstanding_scope(tmp_path):
progress = TaskProgress()
progress.update(plan(), resources={"android", "cua-driver"}, directory=tmp_path)
assert progress.guard("cua-driver")
assert progress.guard("android") is None
assert progress.guard(None) is None # read-only preflight
assert progress.guard(None, completed=True)
assert (tmp_path / "notes/task_progress.md").exists()
with pytest.raises(ValidationError):
progress.update(plan("pending", "active"), resources={"android", "cua-driver"})
omitted = plan()
omitted["stages"] = omitted["stages"][1:]
with pytest.raises(ValueError, match="不能删除"):
progress.update(omitted, resources={"android", "cua-driver"})
progress.update(plan("done", "active"), resources={"android", "cua-driver"})
assert progress.guard("cua-driver") is None
assert progress.guard("android")
progress.update(plan("done", "done"), resources={"android", "cua-driver"})
assert progress.guard(None, completed=True) is None
def test_pc_first_and_intentional_interleaving_are_supported():
progress = TaskProgress()
value = {
"reason": "路由器调整后在手机验证",
"stages": [
{
"id": "mixed",
"scope": "电脑改参数,手机核对,循环验证",
"resources": ["android", "cua-driver"],
"status": "active",
}
],
}
progress.update(value, resources={"android", "cua-driver"})
assert progress.guard("cua-driver") is None
assert progress.guard("android") is None
value["stages"][0]["resources"] = ["cua-driver"]
progress.update(value, resources={"cua-driver"})
assert progress.guard("cua-driver") is None
@pytest.mark.asyncio
async def test_flash_preflight_does_not_authorize_pc_mutation(monkeypatch):
plugins = SimpleNamespace(
packages={"cua-driver": None},
bindings={"plug_pc_look": ("cua-driver", None), "plug_pc_click": ("cua-driver", None)},
tool_names={"plug_pc_look", "plug_pc_click"},
is_read_only=lambda name: name == "plug_pc_look",
)
monkeypatch.setattr("artemis.plugins.runtime.get_plugin_runtime", lambda _: plugins)
runner = FlashRunner.__new__(FlashRunner)
runner.ctx = SimpleNamespace(data_engine=None, execution_target="android")
runner.executor = SimpleNamespace(action_tool_names={"click"})
runner._progress = TaskProgress()
runner._progress.update(plan(), resources={"android", "cua-driver"})
runner._execute_and_record_action = AsyncMock(return_value=(None, "", 0))
runner._refresh_observation = AsyncMock(return_value=(None, ""))
messages = []
await runner._process_tool_calls(
[
{"name": "plug_pc_look", "id": "look", "args": {}},
{"name": "plug_pc_click", "id": "click", "args": {}},
],
SimpleNamespace(indexed_elements=[]),
messages,
"预检",
{},
None,
"",
0,
_TurnRecord(),
)
runner._execute_and_record_action.assert_awaited_once()
assert runner._execute_and_record_action.call_args.args[0] == "plug_pc_look"
assert messages[-1].tool_call_id == "click" and messages[-1].status == "error"
runner._refresh_observation.assert_not_awaited()
@pytest.mark.asyncio
async def test_final_report_reuses_exact_saved_body_and_rejects_missing_note(tmp_path):
runner = FlashRunner.__new__(FlashRunner)
runner.ctx = SimpleNamespace(
data_engine=SimpleNamespace(base_dir=tmp_path, current_step_id="s", record_step=Mock())
)
runner.summarizer = None
runner.final_report_received = False
from artemis.utils.notes import save_verified_note
body = "# 报告\nPC 是深色;手机检查未全部覆盖。"
save_verified_note(tmp_path, "report", body)
messages = []
args = {"status": "completed", "explanation": "全部完成", "report_note_key": "missing"}
result = await runner._finalize_task_report(
"report_task_status", args, "r", "", {}, None, "", messages
)
assert result is None and not runner.final_report_received
args["report_note_key"] = "report"
result = await runner._finalize_task_report(
"report_task_status", args, "r2", "", {}, None, "", messages
)
assert result["explanation"] == body
assert (
runner.ctx.data_engine.record_step.call_args.kwargs["action_taken"]["args"]["explanation"]
== body
)
@@ -99,6 +99,26 @@ def test_video_analyzer_bound_only_when_recording_tools_enabled(mock_context):
HISTORY_TOOLS = {"search_history", "replay_steps", "get_step_screenshot"}
@pytest.mark.parametrize("target", ["android", "computer"])
def test_flash_binds_task_text_delivery_only_when_trace_available(mock_context, tmp_path, target):
from types import SimpleNamespace
mock_context.execution_target = target
with patch("artemis.controllers.unified_controller.get_driver"):
runner = FlashRunner(mock_context, goal="写报告")
tools = runner._get_tools()
assert "save_note" not in {t.name for t in tools}
assert "`save_note`" not in runner._render_system_prompt(tools)
mock_context.data_engine = SimpleNamespace(base_dir=tmp_path)
tools = runner._get_tools()
names = {t.name for t in tools}
assert {"save_note", "read_note", "list_notes"} <= names
save = next(t for t in tools if t.name == "save_note")
assert save.parameters["required"] == ["key", "content"]
prompt = runner._render_system_prompt(tools)
assert "notes/output.md" in prompt and "`save_note`" in prompt
def test_history_tools_bound_only_with_a_data_engine_session(mock_context):
with patch("artemis.controllers.unified_controller.get_driver"):
# No DataEngine: nothing to read, the tools stay out (as in Pro).
+55 -4
View File
@@ -127,23 +127,74 @@ async def test_completed_exploration_plan_does_not_force_an_extra_action(
content="Confirmed home; recorded evidence.",
tool_calls=[{"name": "save_note", "args": {}, "id": "plan"}],
)
invoke = AsyncMock(side_effect=[response, click()])
final = AIMessage(content="# 完整调查报告\n已回主页,证据见步骤 3。未检查其他页面。")
finishing = scenario in {"complete", "unchanged"}
invoke = AsyncMock(side_effect=[response, final if finishing else click()])
monkeypatch.setattr("artemis.agents.operator.operator.acomplete", invoke)
messages = []
result = await node._invoke_llm_loop(
Mock(), messages, [SimpleNamespace(name="save_note")], [], state
)
if scenario == "complete":
if finishing:
assert invoke.await_count == 2
assert result[0] is None
assert final.content in result[1]
assert state.execution_report == final.content
assert any("完整报告正文" in m.content for m in messages if isinstance(m, SystemMessage))
assert result[3] is False
elif scenario == "pending":
assert invoke.await_count == 1
assert result[0] is None
assert result[1] == "Confirmed home; recorded evidence."
assert result[3] is False
assert state.execution_report is None
return
else:
assert invoke.await_count == 2
assert result[0]
assert state.execution_report is None
assert any(isinstance(m, ToolMessage) and m.tool_call_id == "plan" for m in messages)
@pytest.mark.asyncio
async def test_final_report_is_discarded_when_guidance_arrives(operator, monkeypatch):
node, state = operator
state.execution_report = "旧报告"
node.ctx.exploration_id = "exploration"
node._read_task_plan_text = lambda: "- [x] 调查完成"
pending = {"value": False}
monkeypatch.setattr(
"artemis.exploration.runtime.needs_exploration_turn", lambda ctx: pending["value"]
)
publish = Mock()
monkeypatch.setattr("artemis.exploration.runtime.publish_response", publish)
async def answer(*args):
pending["value"] = True
return AIMessage(content="这是过时的最终报告")
monkeypatch.setattr("artemis.agents.operator.operator.acomplete", answer)
result = await node._invoke_llm_loop(Mock(), [], [], [], state)
assert result[0] is None
assert state.execution_report is None
publish.assert_not_called()
@pytest.mark.asyncio
async def test_empty_final_report_uses_bounded_recovery(operator, monkeypatch):
node, state = operator
node.ctx.exploration_id = "exploration"
node._read_task_plan_text = lambda: "- [x] 调查完成"
monkeypatch.setattr("artemis.exploration.runtime.needs_exploration_turn", lambda ctx: False)
monkeypatch.setattr("artemis.exploration.runtime.publish_response", Mock())
monkeypatch.setattr("artemis.agents.operator.operator.get_llm", Mock(return_value=Mock()))
monkeypatch.setattr(settings, "ARTEMIS_TOOL_FALLBACK_MAX_NO_PROGRESS_ATTEMPTS", 2)
invoke = AsyncMock(return_value=AIMessage(content=" "))
monkeypatch.setattr("artemis.agents.operator.operator.acomplete", invoke)
with pytest.raises(OperatorNoProgressError):
await node._invoke_llm_loop(Mock(), [], [], [], state)
assert state.execution_report is None
assert invoke.await_count == 5
@pytest.mark.asyncio
async def test_invalid_target_can_be_corrected_before_limit(operator, monkeypatch):
result, messages, invoke = await run_loop(operator, monkeypatch, [click("14"), click("13")])
@@ -54,8 +54,8 @@ from artemis.memory.transcript import (
# SHA-256 snapshots of the legacy system message with the fixed inputs below.
# Update these when an intentional template change alters the rendered prompt.
GOLDEN_EMPTY_PLAN = "58bf034527b2d782f7746d8e379cb0decc44a05dfa18cb73153d49cd8265002b"
GOLDEN_SENTINEL_PLAN = "b328052390a0a815c096b8ec4c5286421be487e7b6cd3bec55d6bc62a1f2518f"
GOLDEN_EMPTY_PLAN = "e83c2c6cc1e28b04898e6602194cb2fbac67fe5a3aec8e109872c375f4e59944"
GOLDEN_SENTINEL_PLAN = "dba4eb2cdcd9389148ac3228c6e9e69fdbe8d407dec76b58aa65509ad768c392"
SCREENSHOT_B64 = base64.b64encode(b"fake-jpeg-bytes").decode("utf-8")
+101 -4
View File
@@ -17,6 +17,7 @@ settlement timeout, when-semantics at the exit, run outcome, and the four
switch-combination matrix."""
import asyncio
import json
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
@@ -83,7 +84,7 @@ async def test_exploration_question_prevents_completed_plan_from_starting_final_
repo.ask(run["id"], "下一步做什么?", "返回主页", "ask")
with patch("artemis.graph.graph.run_final_check", new=AsyncMock()) as reviewer:
assert convergence_gate(_make_state(), ctx) == "continue"
result = await exit_settlement_node(_make_state(), ctx)
result = await exit_settlement_node(_make_state(execution_report="本次调查报告"), ctx)
assert result == {"exit_settlement_route": "continue"}
reviewer.assert_not_called()
assert repo.get(session["id"])["pending_question"]
@@ -100,7 +101,7 @@ async def test_exploration_late_return_home_guidance_invalidates_final_review(ex
return CheckReport(verdicts=[], unmet_subgoals=[])
with patch("artemis.graph.graph.run_final_check", new=review):
result = await exit_settlement_node(_make_state(), ctx)
result = await exit_settlement_node(_make_state(execution_report="本次调查报告"), ctx)
assert result == {"exit_settlement_route": "continue"}
assert repo.run(run["id"])["status"] == "running"
assert repo.run(run["id"])["user_guidance_count"] == 1
@@ -118,12 +119,99 @@ async def test_review_stops_capture_before_waiting_and_seals_exploration(explora
return CheckReport(verdicts=[], unmet_subgoals=[])
with patch("artemis.graph.graph.run_final_check", new=review):
result = await exit_settlement_node(_make_state(), ctx)
result = await exit_settlement_node(_make_state(execution_report="本次调查报告"), ctx)
assert result["exit_settlement_route"] == "end"
assert repo.get(session["id"])["status"] == "finalizing"
ctx.on_execution_resume.assert_not_called()
@pytest.mark.asyncio
async def test_completed_exploration_waits_for_report_before_review(exploration_exit):
ctx, repo, session, run = exploration_exit
state = State.initial("保存并交付完整报告")
with patch("artemis.graph.graph.run_final_check", new=AsyncMock()) as review:
assert convergence_gate(state, ctx) == "continue"
assert await exit_settlement_node(state, ctx) == {"exit_settlement_route": "continue"}
review.assert_not_called()
ctx.on_execution_review.assert_not_called()
assert repo.run(run["id"])["status"] == "running"
@pytest.mark.asyncio
@pytest.mark.parametrize("verdict_status", ["passed", "failed", "inconclusive"])
async def test_report_is_saved_and_fully_delivered_before_independent_review(
exploration_exit, verdict_status
):
from artemis.runtime.case_result import CaseTestResult
ctx, repo, session, run = exploration_exit
body = "# 本次调查报告\n" + "实际证据及未测范围。" * 2400
state = State(initial_goal="交付完整报告", execution_report=body)
_write_plan(ctx.data_engine.base_dir, "- [x] 调查并交付报告\n - assert: 实际页面结果\n")
async def review(*args, **kwargs):
directory = ctx.data_engine.base_dir
assert (directory / "notes" / "output.md").read_text(encoding="utf-8") == body
assert json.loads((directory / "result.json").read_text(encoding="utf-8")) == {
"result": body
}
delivered = [m for m in repo.messages(session["id"]) if m.get("kind") == "execution_report"]
assert len(delivered) == 3
assert "".join(m["text"] for m in delivered) == body
assert kwargs["execution_report"] == body
assert kwargs["report_delivery"]["exploration_message_sequences"] == [
m["sequence"] for m in delivered
]
return CheckReport(
verdicts=[
CheckVerdict(
item_text="实际页面结果",
kind="assert",
status=verdict_status,
evidence="步骤 1",
)
],
test_result=CaseTestResult(
status=verdict_status, reason="按本次实际证据裁定", evidence=["步骤 1 实际页面"]
),
)
with patch("artemis.graph.graph.run_final_check", new=review):
result = await exit_settlement_node(state, ctx)
assert result["run_outcome"]["test_result"]["status"] == verdict_status
assert repo.run(run["id"])["status"] == "finalizing"
@pytest.mark.asyncio
async def test_report_delivery_is_idempotent_with_review_disabled(exploration_exit):
ctx, repo, session, run = exploration_exit
ctx.execution_setup.disable_checker = True
state = State(initial_goal="调查", execution_report="完整报告正文")
assert convergence_gate(state, ctx) == "exit_settlement"
with patch("artemis.graph.graph.run_final_check", new=AsyncMock()) as review:
await exit_settlement_node(state, ctx)
await exit_settlement_node(state, ctx)
review.assert_not_called()
delivered = [m for m in repo.messages(session["id"]) if m.get("kind") == "execution_report"]
assert len(delivered) == 1
assert delivered[0]["text"] == "完整报告正文"
@pytest.mark.asyncio
async def test_report_write_failure_cannot_release_as_completed(exploration_exit, monkeypatch):
from pathlib import Path
ctx, repo, session, run = exploration_exit
state = State(initial_goal="调查", execution_report="完整报告正文")
monkeypatch.setattr(Path, "replace", MagicMock(side_effect=OSError("disk full")))
with patch("artemis.graph.graph.run_final_check", new=AsyncMock()) as review:
with pytest.raises(OSError, match="disk full"):
await exit_settlement_node(state, ctx)
review.assert_not_called()
assert repo.run(run["id"])["status"] == "running"
assert not list(ctx.data_engine.base_dir.rglob("*.tmp"))
def _make_ctx(tmp_path, **setup_kwargs):
setup_kwargs.setdefault("disable_checker", False)
ctx = MagicMock(spec=ArtemisContext)
@@ -311,7 +399,16 @@ async def test_final_check_receives_ledger_and_at_end_items(tmp_path):
captured = {}
async def fake_final(ctx_arg, goal, plan_text, ledger, check_items, attempt_id=None):
async def fake_final(
ctx_arg,
goal,
plan_text,
ledger,
check_items,
attempt_id=None,
execution_report=None,
report_delivery=None,
):
captured["goal"] = goal
captured["ledger"] = ledger
captured["items"] = list(check_items)
+9 -1
View File
@@ -66,7 +66,15 @@ def shared_mcp_backend(tmp_path, monkeypatch):
key: registration.info[key] for key in ("protocol", "installation", "instance", "pid")
}
server = uvicorn.Server(uvicorn.Config(app, host="127.0.0.1", port=port, log_level="error"))
server = uvicorn.Server(
uvicorn.Config(
app,
host="127.0.0.1",
port=port,
log_level="error",
timeout_graceful_shutdown=5, # Match the user-started service's SSE shutdown budget.
)
)
thread = threading.Thread(target=server.run, kwargs={"sockets": [listener]}, daemon=True)
thread.start()
deadline = time.monotonic() + 8
@@ -48,6 +48,46 @@ def test_helper_tools_are_not_device_actions():
assert not (AGENT_TOOL_NAMES & executor.action_tool_names)
@pytest.mark.asyncio
async def test_flash_saves_and_reads_text_with_verified_receipt_without_device_actions(tmp_path):
import hashlib
import json
executor = _make_executor(agent_name="flash")
executor.ctx.data_engine = Mock(base_dir=tmp_path, current_step_id=None)
state = _state()
result = await executor.execute(
"save_note", {"key": "reusable_case", "content": "# 用例\n检查标题"}, "save", state
)
assert result.status == "success"
receipt = json.loads(result.text_summary)
data = (tmp_path / receipt["path"]).read_bytes()
assert receipt["verified"] is True and receipt["sha256"] == hashlib.sha256(data).hexdigest()
assert receipt["bytes"] == len(data)
read = await executor.execute("read_note", {"key": "reusable_case"}, "read", state)
assert read.status == "success" and "检查标题" in read.text_summary
assert result.screenshot_bytes is None and result.ui_elements_text is None
executor.actuator.execute.assert_not_called()
@pytest.mark.asyncio
@pytest.mark.parametrize("key", ["../outside", "C:\\outside", "output", "output.MD"])
async def test_flash_note_rejects_paths_and_reserved_final_report(tmp_path, key):
executor = _make_executor(agent_name="flash")
executor.ctx.data_engine = Mock(base_dir=tmp_path, current_step_id=None)
result = await executor.execute("save_note", {"key": key, "content": "data"}, "save", _state())
assert result.status == "error"
assert not (tmp_path / "notes").exists()
@pytest.mark.asyncio
async def test_flash_note_does_not_fall_back_to_working_directory_without_trace():
result = await _make_executor(agent_name="flash").execute(
"save_note", {"key": "report", "content": "data"}, "save", _state()
)
assert result.status == "error" and "trace" in result.text_summary
# ---------------------------------------------------------------------------
# ask_explorer
# ---------------------------------------------------------------------------
+3 -1
View File
@@ -56,7 +56,9 @@ SCOPED_PROMPT_FILES = [
# apply_operator_prompt_contract. Anything new landing here should be a conscious
# decision.
#: Backticked identifiers in prompts that are parameters or note keys, not tools.
NON_TOOL_IDENTIFIERS = frozenset({"analysis", "task_plan", "tool_enum", "target_description"})
NON_TOOL_IDENTIFIERS = frozenset(
{"analysis", "task_plan", "tool_enum", "target_description", "report_note_key"}
)
_TOOL_REF = re.compile(r"(?:`([a-z][a-z0-9_]{2,})`|\b([a-z][a-z0-9_]{2,})\()")
+39
View File
@@ -245,6 +245,45 @@ async def test_executor_cancellation_reaches_coroutine():
await executor.stop()
@pytest.mark.asyncio
async def test_sync_tool_runs_once_off_both_event_loops():
import threading
executor = ToolExecutor()
executor.start()
caller = threading.get_ident()
calls = []
def operation(value):
calls.append(threading.get_ident())
return {"accepted": value}
try:
assert await executor.invoke(operation, "job") == {"accepted": "job"}
assert len(calls) == 1
assert calls[0] not in {caller, executor.thread.ident}
finally:
await executor.stop()
@pytest.mark.asyncio
async def test_sync_tool_error_does_not_repeat_the_side_effect():
executor = ToolExecutor()
executor.start()
calls = []
def operation():
calls.append("submitted")
raise ValueError("reply lost")
try:
with pytest.raises(ValueError, match="reply lost"):
await executor.invoke(operation)
assert calls == ["submitted"]
finally:
await executor.stop()
def test_stale_pid_and_wrong_installation_are_not_reused(tmp_path, monkeypatch):
monkeypatch.setattr(tempfile, "tempdir", str(tmp_path))
directory = registry.registry_directory()
@@ -0,0 +1,55 @@
from concurrent.futures import ThreadPoolExecutor
from unittest.mock import Mock
import pytest
from apps.admin_console.services.automation_store import AutomationStore
from mcp_server import task_requests
from mcp_server.tools import task_runner
@pytest.fixture
def receipts(tmp_path, monkeypatch):
store = AutomationStore(tmp_path / "automation.db")
monkeypatch.setattr(task_requests, "store", store)
return store
def test_concurrent_retries_dispatch_once(receipts):
execute = Mock(side_effect=lambda sid: {"trace_id": sid, "status": "queued"})
with ThreadPoolExecutor(max_workers=4) as pool:
results = list(
pool.map(
lambda _: task_requests.run_once("one", {"goal": "observe"}, execute), range(8)
)
)
assert execute.call_count == 1
assert len({result["trace_id"] for result in results}) == 1
assert task_requests.lookup("one")["status"] == "queued"
receipts.save_job({"session_id": results[0]["trace_id"], "status": "completed", "position": 1})
assert task_requests.lookup("one")["status"] == "completed"
assert task_requests.lookup("one")["status_scope"] == "task_queue"
def test_conflicting_content_and_lost_reply_never_repeat(receipts):
execute = Mock(side_effect=ConnectionError("after admission"))
with pytest.raises(ConnectionError):
task_requests.run_once("one", {"goal": "observe"}, execute)
receipt = task_requests.run_once("one", {"goal": "observe"}, execute)
assert receipt["submission_state"] == "unknown"
assert receipt["trace_id"]
assert execute.call_count == 1
with pytest.raises(ValueError, match="不同任务内容"):
task_requests.run_once("one", {"goal": "write"}, execute)
assert task_requests.lookup("missing")["status"] == "not_found"
def test_public_task_entry_uses_receipts_before_dispatch(receipts, monkeypatch):
execute = Mock(
side_effect=lambda **kwargs: {"trace_id": kwargs["trace_id"], "status": "queued"}
)
monkeypatch.setattr(task_runner, "_run_task", execute)
first = task_runner.mobile_run_task("observe", request_id="public-one")
second = task_runner.mobile_run_task("observe", request_id="public-one")
assert first["trace_id"] == second["trace_id"]
execute.assert_called_once()
@@ -72,6 +72,49 @@ def _step(i: int, subgoal: str = "hash-a", **overrides) -> dict:
return step
@pytest.mark.parametrize("minimal", [False, True])
def test_runtime_file_paths_survive_even_minimal_history_folding(minimal):
from artemis.memory.chunking import build_action_ledger
action = {
"action": "plug_pc_save",
"plugin_id": "cua-driver",
"attachments": [
{"path": "artifacts/plugins/cua-driver/footer.png", "sha256": "a" * 64, "bytes": 123}
],
}
text = build_action_ledger([_step(35, action_taken=action)], SESSION_START, minimal=minimal)
assert "Step 35" in text and "footer.png" in text and "a" * 64 in text
def test_default_larger_batches_reduce_capsule_calls_without_losing_step_index():
from artemis.config import MemoryChunkingConfig
current = MemoryChunkingConfig()
steps = [_step(i) for i in range(1, 48)]
counts = []
for minimum, target in [(3, 2000), (current.min_steps, current.target_source_tokens)]:
ledger, chunker, _, capsule = _make(
steps,
min_active=5,
min_steps=minimum,
target_tokens=target,
auto_capsule=True,
meter=lambda: 75_000,
)
_run_turns(ledger, chunker, 1, 47, lambda _: "same-stage")
counts.append(len(capsule.submitted))
consumed = {sid for c in chunker.chunks for sid in c.source_step_ids}
consumed.update(
sid for era in chunker.eras for c in era.chunks for sid in c.source_step_ids
)
remaining = {sid for turn in ledger.unchunked_turns() for sid in turn.get("step_keys", [])}
assert consumed | remaining # remaining live/frozen records are not dropped
for number in range(1, 42):
assert f"s{number}" in consumed | remaining
assert counts[1] < counts[0]
class FakeEngine:
def __init__(self, steps: list[dict], base_dir=None):
self.steps = steps
+5 -5
View File
@@ -249,7 +249,7 @@ def test_cli_mcp_install_codex_preserves_config_and_is_idempotent(tmp_path, monk
config_file.write_text(
'model = "test-model"\n\n'
'[mcp_servers.existing]\ncommand = "echo"\n\n'
'[mcp_servers.artemis]\ncommand = "old-command"\n',
'[mcp_servers.artemis]\ncommand = "old-command"\nargs = ["-m", "mcp_server"]\n',
encoding="utf-8",
)
@@ -283,12 +283,12 @@ def test_cli_mcp_install_codex_preserves_config_and_is_idempotent(tmp_path, monk
"mobile_export_report",
]
assert data["mcp_servers"]["artemis"]["env"]["PYTHONPATH"] == str(tmp_path)
assert config_text.count("# BEGIN ARTEMIS MCP CONFIG") == 1
assert config_text.count("[mcp_servers.artemis]") == 1
agents_file = codex_dir / "AGENTS.md"
agents_text = agents_file.read_text(encoding="utf-8")
assert "Mobile Testing Mindset" in agents_text
assert agents_text.count("<!-- BEGIN ARTEMIS MOBILE TESTING RULES -->") == 1
assert not agents_file.exists()
skill_file = tmp_path / ".agents/skills/artemis-use/SKILL.md"
assert "name: artemis-use" in skill_file.read_text(encoding="utf-8")
def test_cli_restart_help():
+124
View File
@@ -0,0 +1,124 @@
import json
from pathlib import Path
from unittest.mock import Mock
import pytest
from artemis.integrations import codex_skill as integration
@pytest.fixture
def bundle(tmp_path, monkeypatch):
root = tmp_path / "source"
source = root / "skills" / "artemis-use"
source.mkdir(parents=True)
(source / "SKILL.md").write_text(
"---\nname: artemis-use\ndescription: official\n---\n", "utf-8"
)
(source / "references").mkdir()
(source / "references" / "modes.md").write_text("official modes", "utf-8")
codex_home = tmp_path / "用户配置" / ".codex"
monkeypatch.setenv("CODEX_HOME", str(codex_home))
return root, source, codex_home
def test_install_overwrites_entire_reserved_directory_and_is_noop_when_unchanged(bundle):
root, source, home = bundle
target = home / "skills" / "artemis-use"
target.mkdir(parents=True)
(target / "SKILL.md").write_text("unmanaged custom content", "utf-8")
(target / "custom.txt").write_text("must be removed", "utf-8")
unrelated = home / "skills" / "personal" / "SKILL.md"
unrelated.parent.mkdir()
unrelated.write_text("personal", "utf-8")
assert integration.sync_official_skill(root)["status"] == "updated"
assert not (target / "custom.txt").exists()
assert (target / "SKILL.md").read_bytes() == (source / "SKILL.md").read_bytes()
assert unrelated.read_text("utf-8") == "personal"
before = (target / "SKILL.md").stat().st_mtime_ns
assert integration.sync_official_skill(root)["status"] == "unchanged"
assert (target / "SKILL.md").stat().st_mtime_ns == before
(target / "SKILL.md").write_text("edited", "utf-8")
assert integration.sync_official_skill(root)["status"] == "updated"
assert (target / "SKILL.md").read_bytes() == (source / "SKILL.md").read_bytes()
def test_update_removes_retired_files_and_migrates_only_marked_rules(bundle):
root, source, home = bundle
integration.sync_official_skill(root)
original = "user prefix\n<!-- BEGIN ARTEMIS MOBILE TESTING RULES -->\nold\n<!-- END ARTEMIS MOBILE TESTING RULES -->\nuser suffix\n"
for name in ("AGENTS.md", "AGENTS.override.md"):
(home / name).write_text(original, "utf-8")
(source / "references" / "modes.md").unlink()
integration.sync_official_skill(root)
assert not (home / "skills/artemis-use/references/modes.md").exists()
for name in ("AGENTS.md", "AGENTS.override.md"):
assert (home / name).read_text("utf-8") == "user prefix\nuser suffix\n"
assert (home / (name + ".artemis-update.bak")).read_text("utf-8") == original
def test_failed_replacement_restores_previous_bundle(bundle, monkeypatch):
root, source, home = bundle
integration.sync_official_skill(root)
target = home / "skills/artemis-use"
original = (target / "SKILL.md").read_bytes()
(source / "SKILL.md").write_text("new version", "utf-8")
rename = Path.rename
def fail_staging(path, destination):
if path.name.startswith(".artemis-use-stage-"):
raise PermissionError("held open")
return rename(path, destination)
monkeypatch.setattr(Path, "rename", fail_staging)
assert integration.sync_skill_safely(root)["status"] == "error"
assert (target / "SKILL.md").read_bytes() == original
assert not list(target.parent.glob(".artemis-use-stage-*"))
def test_newer_official_bundle_is_not_downgraded(bundle):
root, _, home = bundle
integration.sync_official_skill(root)
stamp = home / "skills/artemis-use" / integration.STAMP
data = json.loads(stamp.read_text("utf-8"))
data["artemis_version"] = "99.0.0"
stamp.write_text(json.dumps(data), "utf-8")
assert integration.sync_official_skill(root)["status"] == "newer_installed"
def test_default_directory_and_duplicate_cleanup(bundle, tmp_path, monkeypatch):
root, source, _ = bundle
monkeypatch.delenv("CODEX_HOME")
monkeypatch.setattr(Path, "home", lambda: tmp_path)
legacy = tmp_path / ".codex/skills/artemis-use"
legacy.mkdir(parents=True)
(legacy / "SKILL.md").write_text("old", "utf-8")
assert integration.sync_official_skill(root)["path"] == str(legacy)
modern = tmp_path / ".agents/skills/artemis-use"
modern.mkdir(parents=True)
(modern / "SKILL.md").write_text("duplicate", "utf-8")
assert integration.sync_official_skill(root)["path"] == str(modern)
assert not legacy.exists()
def test_actual_bundle_links_are_self_contained():
import re
source = Path(__file__).resolve().parents[2] / "skills/artemis-use"
for document in source.rglob("*.md"):
for relative in re.findall(r"\]\(([^)]+)\)", document.read_text("utf-8")):
if "://" in relative or relative.startswith("#"):
continue
target = (document.parent / relative.split("#", 1)[0]).resolve()
assert target.is_relative_to(source.resolve()), (document, relative)
assert target.exists(), (document, relative)
def test_malformed_stamp_does_not_block_authoritative_update(bundle):
root, source, home = bundle
integration.sync_official_skill(root)
target = home / "skills/artemis-use"
(target / integration.STAMP).write_text("[]", "utf-8")
(target / "SKILL.md").write_text("unofficial", "utf-8")
assert integration.sync_official_skill(root)["status"] == "updated"
assert (target / "SKILL.md").read_bytes() == (source / "SKILL.md").read_bytes()
@@ -0,0 +1,39 @@
"""The default SDK path must return the answer already prepared for review."""
import json
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
from artemis.config import OutputConfig
from artemis.graph.state import State
from artemis.sdk.agent import Agent
@pytest.mark.asyncio
@pytest.mark.parametrize("outputter_enabled", [True, False])
async def test_default_output_reuses_executor_report_without_another_model(
tmp_path, monkeypatch, outputter_enabled
):
synthesize = AsyncMock()
monkeypatch.setattr("artemis.sdk.agent.outputter", synthesize)
ctx = SimpleNamespace(
execution_setup=SimpleNamespace(
outputter=SimpleNamespace(enabled=outputter_enabled, force_synthesis=False),
disable_outputter=not outputter_enabled,
)
)
result_path = tmp_path / "result.json"
body = "# 调查结果\n双端检查已完成;未测功能不能记为通过。"
result = await Agent._extract_output(
object.__new__(Agent),
task_name="exploration",
ctx=ctx,
request=SimpleNamespace(llm_output_path=result_path),
output_config=OutputConfig(structured_output=None, output_description=None),
state=State(initial_goal="调查", execution_report=body),
)
assert result == body
assert json.loads(result_path.read_text(encoding="utf-8")) == {"result": body}
synthesize.assert_not_called()
+32
View File
@@ -232,6 +232,38 @@ async def test_rerun_includes_user_edited_preconditions(service, monkeypatch):
assert "已连接测试网络" in enqueue.call_args.args[0].goal
@pytest.mark.asyncio
@pytest.mark.parametrize("field", ["goal", "preconditions"])
async def test_unresolved_case_never_reserves_run_or_creates_batch(service, monkeypatch, field):
from apps.admin_console.services import automation_service
enqueue = AsyncMock()
monkeypatch.setattr("apps.admin_console.routers.tasks.run_task", enqueue)
monkeypatch.setattr(automation_service, "store", service.repository.database)
session = service.create(CreateExploration(goal="探索", request_id="create"))
draft = {"name": "旧版用例", "goal": "检查页面", field: "报告保存到 ${报告位置}"}
service.repository.save_case(session["id"], CaseDraft(**draft), 0, "save")
with pytest.raises(ValueError, match=r"未填写参数.*\$\{报告位置\}"):
await service.validate(session["id"], ValidateCaseRequest(request_id="validate", version=1))
with pytest.raises(ValueError, match="未填写参数"):
service.create_case_batch(session["id"], CaseBatchRequest(request_id="batch", version=1))
enqueue.assert_not_called()
assert service.repository.runs(session["id"]) == []
assert service.repository.database.records("batch") == []
assert service.repository.case(session["id"])["version"] == 1
# A saved correction can use the same admission request ID; rejection created no run.
service.repository.save_case(
session["id"],
CaseDraft(name="可执行", goal="检查页面,报告保存到本次任务 trace"),
1,
"edit",
)
enqueue.return_value = {"status": "queued"}
await service.validate(session["id"], ValidateCaseRequest(request_id="validate", version=2))
enqueue.assert_awaited_once()
@pytest.mark.parametrize("profile", ["flash", "pro"])
def test_case_batch_creates_only_an_idempotent_draft_with_bound_resources(
service, monkeypatch, profile
+44
View File
@@ -82,6 +82,50 @@ async def test_generation_is_idempotent_and_preserves_rerun_independence(synthes
model.with_structured_output.return_value.ainvoke.assert_awaited_once()
@pytest.mark.asyncio
async def test_synthesis_repairs_unbound_parameters_once_with_original_evidence(synthesis):
repository, session, run, model = synthesis
invoke = model.with_structured_output.return_value.ainvoke
invoke.side_effect = [
CaseDraft(name="错误草稿", goal="访问 ${首页URL}", preconditions="已指定 ${报告位置}"),
CaseDraft(name="修订", goal="核对设置,报告保存在本次任务 trace。", evidence=["步骤3"]),
]
result = await generate_case(
repository, session["id"], "generate", run["id"], 0, ctx=SimpleNamespace(task_control=None)
)
assert result["version"] == 1 and "${" not in result["goal"]
assert invoke.await_count == 2
messages = invoke.await_args.args[0]
assert "不生成 ${名称}" in messages[0].content
assert "用户明确指定的位置必须保留" in messages[0].content
assert "不要让一次复跑递归地产生下一份复跑用例" in messages[0].content
assert "保留该业务验收" in messages[0].content
assert "设置开启" in messages[1].content
assert "不得猜测未知业务输入或降低验收标准" in messages[-1].content
@pytest.mark.asyncio
async def test_failed_parameter_repair_never_saves_unexecutable_generated_case(synthesis):
repository, session, run, model = synthesis
repository.save_case(session["id"], CaseDraft(name="原用例", goal="核对设置"), 0, "save")
invoke = model.with_structured_output.return_value.ainvoke
invoke.return_value = CaseDraft(name="错误", goal="保存到 ${报告位置}")
with pytest.raises(ValueError, match="未填写参数"):
await generate_case(
repository,
session["id"],
"generate",
run["id"],
1,
ctx=SimpleNamespace(task_control=None),
)
assert invoke.await_count == 2
assert repository.case(session["id"])["version"] == 1
assert repository.case(session["id"])["name"] == "原用例"
assert repository.database.records("exploration_generation")[0]["status"] == "failed"
assert repository.run(run["id"])["outcome"]["test_result"]["status"] == "passed"
@pytest.mark.asyncio
async def test_model_generation_cannot_overwrite_a_concurrent_manual_revision(synthesis):
repository, session, run, model = synthesis
+10 -7
View File
@@ -132,7 +132,7 @@ command = "leave-alone"
assert config["disabled_tools"] == ["mobile_config"]
assert config["tool_timeout_sec"] == 900
assert config["env"]["OPENAI_API_KEY"] == "test-only-secret"
assert config["enabled_tools"] == ["mobile_run_task", "mobile_batch"]
assert config["enabled_tools"] == ["mobile_run_task"]
assert data["mcp_servers"]["other"]["command"] == "leave-alone"
assert not update.refresh_codex(path, root, "new-python", ("mobile_run_task", "mobile_batch"))
assert path.with_name(path.name + ".artemis-update.bak").read_text(encoding="utf-8") == original
@@ -178,7 +178,7 @@ def test_codex_sync_preserves_tool_policies_verbatim(tmp_path, monkeypatch, poli
"PYTHONUNBUFFERED": "1",
"ARTEMIS_DESKTOP_NOTIFY": "true",
}
assert entry["enabled_tools"] == list(mcp_command.ARTEMIS_MCP_TOOLS)
assert entry["enabled_tools"] == ["mobile_run_task"]
assert entry["disabled_tools"] == ["mobile_config"]
assert entry["tools"] == before["mcp_servers"]["artemis"]["tools"]
assert policies.split("\n", 1)[1] in refreshed
@@ -199,7 +199,10 @@ def test_codex_sync_reports_generated_toml_error_without_blame_or_overwrite(
path.write_text(original, encoding="utf-8")
monkeypatch.setenv("CODEX_HOME", str(codex_home))
monkeypatch.setenv("ARTEMIS_LAUNCH_LANG", "zh-CN")
monkeypatch.setattr(update, "_replace_toml_tables", lambda *_: 'tools = {"secret": true}\n')
monkeypatch.setattr(
"artemis.integrations.codex_mcp._replace_toml_tables",
lambda *_: 'tools = {"secret": true}\n',
)
assert update.refresh_mcp(tmp_path / "project") == 1
assert path.read_text(encoding="utf-8") == original
assert not path.with_name(path.name + ".artemis-update.bak").exists()
@@ -252,7 +255,7 @@ def test_mcp_sync_updates_only_codex_and_preserves_other_clients(tmp_path, monke
assert entry["enabled"] is False
assert entry["env"]["OPENAI_API_KEY"] == "keep-secret"
assert entry["env"]["PYTHONPATH"] == str(root)
assert entry["enabled_tools"] == list(mcp_command.ARTEMIS_MCP_TOOLS)
assert entry["enabled_tools"] == ["mobile_run_task", "retired_tool"]
cursor = json.loads((cursor_home / "mcp.json").read_text(encoding="utf-8"))
assert "artemis" not in cursor["mcpServers"]
assert cursor["mcpServers"]["other"] == {"command": "keep"}
@@ -331,9 +334,9 @@ def test_codex_sync_reports_missing_directory_without_touching_other_clients(
assert update.refresh_mcp(tmp_path / "project") == 0
output = capsys.readouterr().out
assert str(tmp_path / ".codex/config.toml") in output
assert "未发现 Codex 配置目录" in output
assert "--install codex" in output
assert not (tmp_path / ".codex").exists()
assert "已安装" in output
assert (tmp_path / ".codex/config.toml").is_file()
assert (tmp_path / ".agents/skills/artemis-use/SKILL.md").is_file()
def test_custom_codex_command_is_preserved_and_reported(tmp_path, monkeypatch, capsys):
+2 -1
View File
@@ -319,7 +319,8 @@ async def test_mid_stream_failure_records_stream_reset_payload(monkeypatch):
assert payload["action"] == "discard"
assert payload["reason"] == "mid_stream_failure"
assert "stream_exec_id" in payload
assert "lower API priority" in payload["message"]
assert "未完成的输出已丢弃" in payload["message"]
assert "lower API priority" not in payload["message"]
@pytest.mark.asyncio
+141
View File
@@ -0,0 +1,141 @@
"""Runaway click arguments must never reach the executor or endless retries."""
from copy import deepcopy
import json
from types import SimpleNamespace
from unittest.mock import AsyncMock
from langchain_core.messages import AIMessageChunk
import pytest
from artemis.llm.reliability import FailureCategory, LLMExhaustedError, retry_policy_for
from artemis.llm.router import ModelEndpoint
from artemis.llm.stream_guard import (
SMALL_ACTION_ARGUMENT_LIMIT,
ToolArgumentGuard,
ToolArgumentOverflow,
)
from artemis.services import llm
def delta(arguments, *, name=None, index=0):
return AIMessageChunk(
content=[],
tool_call_chunks=[{"name": name, "args": arguments, "index": index}],
)
@pytest.fixture(autouse=True)
def isolate(monkeypatch, tmp_path):
llm._ENDPOINT_BREAKER._states.clear()
llm._NON_STREAMING_ENDPOINTS.clear()
monkeypatch.setattr(llm, "PAUSE_FILE", tmp_path / "pause")
monkeypatch.setattr(
llm, "retry_policy_for", lambda _: SimpleNamespace(max_attempts=2, delay_for=lambda _: 0)
)
monkeypatch.setattr(llm, "_wait_for_resume", AsyncMock(return_value=False))
monkeypatch.setattr(llm, "_CURRENT_DATA_ENGINE", None)
@pytest.mark.parametrize("name", ["save_note", "input_text", "report_task_status", "plugin_tool"])
def test_free_text_tools_and_regular_text_are_not_limited(name):
guard = ToolArgumentGuard()
text = "报告内容" * SMALL_ACTION_ARGUMENT_LIMIT
guard.observe(AIMessageChunk(content=text))
guard.observe(delta(json.dumps({"text": text}, ensure_ascii=False), name=name))
def test_split_and_interleaved_arguments_are_counted_per_call_without_storing_body():
guard = ToolArgumentGuard()
for index in (0, 1):
guard.observe(delta('{"target_description":"', name="click", index=index))
for index in (0, 1):
guard.observe(delta("private" * 1200, index=index))
with pytest.raises(ToolArgumentOverflow) as caught:
guard.observe(delta("private" * 1200, index=1))
assert caught.value.details["tool"] == "click"
assert caught.value.details["argument_chars"] > SMALL_ACTION_ARGUMENT_LIMIT
assert "private" not in str(caught.value)
assert "private" not in repr(guard.calls)
def test_name_arriving_after_arguments_still_triggers_guard():
guard = ToolArgumentGuard()
guard.observe(delta("x" * (SMALL_ACTION_ARGUMENT_LIMIT + 1)))
with pytest.raises(ToolArgumentOverflow):
guard.observe(delta("", name="click"))
def test_invalid_output_retry_policy_is_bounded_and_honors_zero_retries(monkeypatch):
monkeypatch.setattr(llm.settings, "OPENAI_MAX_RETRIES", 5)
assert retry_policy_for(FailureCategory.INVALID_OUTPUT).max_attempts == 2
monkeypatch.setattr(llm.settings, "OPENAI_MAX_RETRIES", 0)
assert retry_policy_for(FailureCategory.INVALID_OUTPUT).max_attempts == 1
@pytest.mark.asyncio
@pytest.mark.parametrize("recover", [False, True])
@pytest.mark.parametrize("first_chunk_overflows", [False, True])
async def test_gateway_closes_bad_stream_discards_partial_and_bounds_retry(
monkeypatch, recover, first_chunk_overflows
):
records, resets, closed = [], [], []
class Engine:
current_session_id = "test-session"
current_step_id = None
def record_trace(self, **record):
records.append(deepcopy(record))
def _publish(self, *_):
pass
class Model:
calls = 0
async def astream(self, *_args, **_kwargs):
self.calls += 1
try:
if recover and self.calls == 2:
yield delta('{"target":3}', name="click")
return
if not first_chunk_overflows:
yield delta('{"target_description":"', name="click")
yield delta(
"private" * (SMALL_ACTION_ARGUMENT_LIMIT // 7 + 1),
name="click" if first_chunk_overflows else None,
)
pytest.fail("Runaway stream was consumed after the safety bound")
finally:
closed.append(self.calls)
monkeypatch.setattr(llm, "_CURRENT_DATA_ENGINE", Engine())
monkeypatch.setattr(llm, "_record_llm_event", lambda *a, **_: resets.append(a))
model = Model()
wrapper = llm.RobustChatModelWrapper(model, endpoint=ModelEndpoint(max_retries=5))
if recover:
response = await wrapper.complete([])
assert response.tool_calls == [
{"name": "click", "args": {"target": 3}, "id": None, "type": "tool_call"}
]
llm._wait_for_resume.assert_not_awaited()
else:
with pytest.raises(LLMExhaustedError) as caught:
await wrapper.complete([])
assert caught.value.failure.category is FailureCategory.INVALID_OUTPUT
llm._wait_for_resume.assert_awaited_once()
assert model.calls == 2
assert closed == [1, 2]
failure = next(
r["payload"]
for r in records
if r["type"] == "llm_diagnostic" and r["payload"]["phase"] == "failed"
)
assert failure["stream_guard"]["tool"] == "click"
assert failure["stream_guard"]["argument_chars"] > SMALL_ACTION_ARGUMENT_LIMIT
assert failure["failure_phase"] == "generating_tool_arguments"
assert failure["category"] == "invalid_output"
assert failure["usage"] is None
assert "private" not in json.dumps(records + resets, default=str)
assert any(name == "llm_stream_reset" and p["action"] == "discard" for name, p in resets)
+1 -1
View File
@@ -45,7 +45,7 @@ def test_memory_defaults():
assert cfg.memory.transcript.xml_scrub_depth == 1
# Ready chunks replace their turns only once the
# operator's measured context reaches budget*start_ratio.
assert cfg.memory.chunking.min_steps == 3
assert cfg.memory.chunking.min_steps == 6
assert cfg.memory.transcript.image_scrub_depth_relaxed == 6
assert cfg.memory.transcript.start_ratio == 0.35
assert cfg.memory.transcript.soft_ratio == 0.7
+56
View File
@@ -19,9 +19,65 @@ from artemis.data_engine.models import ImageRecord, SessionMetadata, StepRecord
from artemis.data_engine.storage import StorageManager
from artemis.memory.transcript import TranscriptLedger
from artemis.plugins.results import plugin_result_messages
from artemis.plugins.models import PluginCallResult
from artemis.plugins.runtime import PluginRuntime
from artemis.tools.history.screenshot import load_step_screenshot
def test_large_tree_deduplicates_without_mutating_raw_evidence():
tree = "- Button abc\n" * 3000
result = PluginCallResult(
status="success",
plugin_id="test",
content=[{"type": "text", "text": "Screenshot\n" + tree}],
structured_content={
"tree_markdown": tree,
"capture_id": "capture-current",
"window_bounds": {"width": 2560},
},
)
text = result.text()
assert text.count(tree) == 1
assert result.structured_content["tree_markdown"] == tree
assert "capture-current" in text and "2560" in text
@pytest.mark.parametrize("mode", ["flash", "pro"])
def test_verified_receipts_survive_observation_truncation_for_both_modes(mode):
receipt = {
"path": "artifacts/plugins/test/screen.png",
"sha256": "a" * 64,
"bytes": 123,
"media_type": "image/png",
}
result = PluginCallResult(
status="success",
plugin_id="test",
content=[{"type": "text", "text": "TREE " * 10000}],
attachments=[receipt],
)
if mode == "flash":
blocks = PluginRuntime.content_blocks(result)
else:
action = _action("call", attachments=[receipt])
action["result"] = result.text()
blocks = plugin_result_messages({"execution": [action]}, None)["call"][0].content
text = str(blocks)
assert receipt["path"] in text and receipt["sha256"] in text
assert "verified file receipts" in text
assert "观察文本已节选" in text
assert len(text) < 14000
def test_plugin_claim_does_not_become_runtime_receipt():
result = PluginCallResult(
status="success",
plugin_id="test",
structured_content={"attachments": [{"path": "pretend.png", "verified": True}]},
)
assert "verified file receipts" not in str(PluginRuntime.content_blocks(result))
def _action(call_id, name="plug_test_get_app_state", **kwargs):
return dict(
action=name,
+257
View File
@@ -0,0 +1,257 @@
"""Read-only diagnostics and destructive-action gates against disposable task data."""
import json
from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock
from uuid import uuid4
from fastapi import FastAPI
from fastapi.testclient import TestClient
import pytest
from apps.admin_console.database.repositories.session_repository import SessionRepository
from apps.admin_console.routers import sessions
from apps.admin_console.services import report_export, task_deletion, task_inspection
from apps.admin_console.services.automation_store import AutomationStore
from apps.admin_console.services.request_diagnostics import request_diagnostics
from artemis.data_engine.storage import StorageManager
from artemis.runtime import trace_store
from mcp_server.tools import automation, exploration, inspect_trace, task_manager
@pytest.fixture
def task(tmp_path, monkeypatch):
sid = str(uuid4())
db = tmp_path / "data_engine.db"
StorageManager(db, tmp_path)
store = AutomationStore(db)
repo = SessionRepository(db)
with store.connection() as connection:
connection.execute(
"INSERT INTO sessions (session_id, initial_goal, start_time, status, device_info) "
"VALUES (?, 'observe', 1, 'completed', '{}')",
(sid,),
)
connection.commit()
monkeypatch.setattr(task_inspection, "session_repo", repo)
monkeypatch.setattr(task_inspection, "store", store)
monkeypatch.setattr(task_inspection.evidence_service, "task_root", lambda _: tmp_path / sid)
monkeypatch.setattr(trace_store, "TRACES_DIR", str(tmp_path))
monkeypatch.setattr(report_export, "store", store)
monkeypatch.setattr(report_export, "TRACES_PATH", tmp_path)
(tmp_path / sid).mkdir()
return sid, db, tmp_path, store
def add_trace(store, sid, kind, name, payload, timestamp=1):
with store.connection() as connection:
connection.execute(
"INSERT INTO traces (trace_id,session_id,type,name,payload,timestamp) VALUES (?,?,?,?,?,?)",
(str(uuid4()), sid, kind, name, json.dumps(payload), timestamp),
)
connection.commit()
def test_usage_uses_input_denominator_and_does_not_invent_missing_model(task):
sid, _, _, store = task
assert task_inspection.inspect_task(sid, "usage")["cache_hit_rate"] is None
add_trace(
store,
sid,
"llm_call",
"llm_usage",
{
"node": "operator",
"model": "model-a",
"prompt_tokens": 1000,
"cached_tokens": 900,
"completion_tokens": 500,
},
)
add_trace(
store,
sid,
"llm_call",
"llm_usage",
{
"node": "checker",
"prompt_tokens": 1000,
"cached_tokens": 0,
"completion_tokens": 500,
},
)
result = task_inspection.inspect_task(sid, "usage")
assert result["cache_hit_rate"] == 0.45
assert result["total_tokens"] == 3000
assert result["by_node_model"][1]["model"] == "unknown"
def test_log_tail_is_bounded_paginated_and_redacted(task, monkeypatch):
sid, _, root, _ = task
monkeypatch.setenv("TEST_API_KEY", "test-only-sensitive-value")
lines = ["x" * 200 for _ in range(1000)] + [
"old",
"test-only-sensitive-value",
"Bearer abcdef",
"password=hush",
]
(root / sid / "stderr.log").write_text("\n".join(lines), "utf-8")
result = task_inspection.inspect_task(sid, "logs", limit=2)
error = result["logs"][1]
assert error["older_bytes_omitted"] > 0
assert error["next_offset"] == 2
assert error["lines"] == ["Bearer [REDACTED]", "password=[REDACTED]"]
older = task_inspection.inspect_task(sid, "logs", offset=2, limit=2)
assert older["logs"][1]["lines"] == ["old", "[REDACTED]"]
assert result["logs"][0]["available"] is False
def test_report_pagination_preserves_total_and_caps_nested_content(task, monkeypatch):
sid, _, _, _ = task
monkeypatch.setattr(
report_export,
"session_report",
lambda _: {
"steps": [{"number": i} for i in range(35)],
"content": "x" * 100000,
"metadata": {"api_key": "hidden"},
"nested": [[{"value": "x" * 100} for _ in range(1000)] for _ in range(1000)],
},
)
result = task_inspection.inspect_task(sid, "report", offset=20, limit=10)
assert result["steps"]["items"] == [{"number": i} for i in range(20, 30)]
assert result["steps"]["total"] == 35
assert result["steps"]["next_offset"] == 30
assert result["metadata"]["api_key"] == "[REDACTED]"
assert result["content_truncated"] is True
assert len(json.dumps(result, ensure_ascii=False)) < 150000
def test_request_pagination_reports_latest_phase_only(task):
sid, db, _, store = task
with store.connection() as connection:
connection.execute("UPDATE sessions SET status='running' WHERE session_id=?", (sid,))
connection.commit()
add_trace(
store, sid, "llm_diagnostic", "request", {"request_id": "r", "phase": "waiting_response"}, 1
)
add_trace(store, sid, "llm_diagnostic", "request", {"request_id": "r", "phase": "completed"}, 2)
assert request_diagnostics(sid, db_path=db, limit=1)["next_offset"] == 1
older = request_diagnostics(sid, db_path=db, limit=1, offset=1)
assert older["requests"][0]["active"] is False
assert older["next_offset"] is None
def test_status_and_report_use_same_human_review_without_losing_ai_result(task):
sid, db, root, store = task
(root / sid / "run_outcome.json").write_text(
json.dumps(
{
"test_result": {
"status": "failed",
"reason": "AI found mismatch",
"evidence": ["step 1: actual differs from expected"],
},
}
),
"utf-8",
)
store.put(
"case_review",
sid,
{"reviews": [{"status": "passed", "reason": "用户确认", "reviewed_at": 10}]},
)
result = task_manager.mobile_manage_task("status", sid)["test_result"]
assert result == report_export.session_report(sid, db_path=db)["test_result"]
assert result["source"] == "human"
assert result["ai_result"]["status"] == "failed"
def test_inspection_not_found_and_pagination_validation(task):
app = FastAPI()
app.include_router(sessions.router)
client = TestClient(app)
assert client.get(f"/api/sessions/{uuid4()}/inspection").status_code == 404
assert client.get(f"/api/sessions/{task[0]}/inspection?limit=101").status_code == 422
def test_active_normal_task_delete_is_rejected_without_touching_history(task, monkeypatch):
sid, _, root, store = task
monkeypatch.setattr(task_deletion, "session_repo", task_inspection.session_repo)
monkeypatch.setattr(task_deletion, "store", store)
monkeypatch.setattr(task_deletion, "_check_runs", lambda _: None)
store.save_job({"session_id": sid, "status": "running", "position": 1})
app = FastAPI()
app.include_router(sessions.router)
client = TestClient(app)
assert client.get(f"/api/sessions/{sid}/deletion-preview").status_code == 409
assert client.post(f"/api/sessions/{sid}/delete").status_code == 409
assert (root / sid).is_dir()
assert store.job(sid)["status"] == "running"
def test_mcp_review_and_delete_require_user_intent_and_share_http_backend(monkeypatch):
request = AsyncMock(return_value={"test_result": {"source": "human"}})
monkeypatch.setattr(automation, "_request", request)
for action in ("review", "delete"):
with pytest.raises(ValueError, match="用户"):
task_manager.mobile_manage_task(action, "task")
request.assert_not_awaited()
result = task_manager.mobile_manage_task(
"review", "task", review_status="passed", review_reason="确认", user_confirmed=True
)
assert result["test_result"]["source"] == "human"
request.assert_awaited_once_with(
"PUT", "/api/automation/reports/task/review", json={"status": "passed", "reason": "确认"}
)
@pytest.mark.asyncio
async def test_mcp_read_inspection_and_exploration_delete_routes(monkeypatch):
request = AsyncMock(return_value={"ok": True})
monkeypatch.setattr(automation, "_request", request)
monkeypatch.setattr(exploration, "_request", request)
await inspect_trace.mobile_inspect_trace("view_requests", "task", offset=20, limit=10)
request.assert_awaited_with(
"GET",
"/api/sessions/task/inspection",
params={"view": "requests", "offset": 20, "limit": 10},
)
with pytest.raises(ValueError, match="用户"):
await exploration.mobile_exploration("delete", exploration_id="e")
await exploration.mobile_exploration("delete", exploration_id="e", user_confirmed=True)
request.assert_awaited_with("DELETE", "/api/explorations/e")
@pytest.mark.asyncio
async def test_plugin_probe_only_targets_installed_plugin(monkeypatch):
async def request(method, path, **kwargs):
return {"plugins": [{"manifest": {"id": "demo"}}]} if path == "/api/plugins" else {}
mocked = AsyncMock(side_effect=request)
monkeypatch.setattr(automation, "_request", mocked)
result = await automation.mobile_workspace(
include_devices=False, plugin_id="demo", probe_plugin=True, include_development=True
)
assert result["plugins"][0]["manifest"]["id"] == "demo"
assert result["development"]["plugin_schema_version"] == 2
assert [call.args for call in mocked.call_args_list if call.args[0] == "POST"] == [
("POST", "/api/plugins/demo/diagnose")
]
def test_exploration_deletion_preview_keeps_validation_runs(monkeypatch):
repository = SimpleNamespace(
get=lambda _: {},
runs=lambda _: [
{"id": "explore-run", "kind": "exploration"},
{"id": "rerun", "kind": "validation"},
],
)
monkeypatch.setattr(task_deletion, "explorations", repository)
check = Mock()
monkeypatch.setattr(task_deletion, "_check_runs", check)
result = task_deletion.exploration_deletion_preview("e")
assert result["session_ids"] == ["explore-run"]
assert result["preserved_session_ids"] == ["rerun"]
check.assert_called_once()
+23
View File
@@ -17,11 +17,34 @@ from artemis.utils.notes import (
list_notes_info,
read_note_content,
save_note_content,
save_verified_note,
update_note_content,
)
import pytest
@pytest.mark.parametrize("content", ["# 报告\n结果\n", "line\r\nsecond", ""])
def test_verified_note_receipt_matches_actual_file_bytes(tmp_path, content):
import hashlib
receipt = save_verified_note(tmp_path, "report.md", content)
data = (tmp_path / receipt["path"]).read_bytes()
assert receipt["path"] == "notes/report.md"
assert receipt["verified"] is True and receipt["bytes"] == len(data)
assert receipt["sha256"] == hashlib.sha256(data).hexdigest()
def test_verified_note_does_not_confirm_incomplete_write(tmp_path, monkeypatch):
from artemis.utils import notes
def partial_write(base_dir, key, content):
save_note_content(base_dir, key, content[:1])
monkeypatch.setattr(notes, "save_note_content", partial_write)
with pytest.raises(OSError, match="读回不一致"):
save_verified_note(tmp_path, "report", "完整报告")
def test_list_notes_info(tmp_path):
base_dir = tmp_path