feat: add Android task file evidence and finalize plugin design

This commit is contained in:
chenmingxuan committed 2026-10-15 18:27:11 +08:00
1 parent 96fedc816f
commit 72ff342605
20 files changed
+1005 -35

No files matched your search

@@ -988,6 +988,10 @@ def test_task_material_zip_contains_ordered_evidence_and_reports(
):
job = isolated.jobs(service.create_batch(payload())["id"])[0]
isolated.save_job({**job, "case": {**job["case"], "case_id": "1.1"}})
artifact_dir = reports.TRACES_PATH / job["session_id"] / "artifacts" / "android"
artifact_dir.mkdir(parents=True)
(artifact_dir / "安卓截图.png").write_bytes(b"native png")
(artifact_dir / ".artemis-partial.tmp").write_bytes(b"incomplete")
evidence = tmp_path / "evidence"
evidence.mkdir()
first = evidence / "first.png"
@@ -1015,9 +1019,37 @@ def test_task_material_zip_contains_ordered_evidence_and_reports(
assert archive.read("1.1/任务录屏/0001_recording.mp4") == b"recording.mp4"
manifest = json.loads(archive.read("1.1/文件清单.json"))
assert manifest["session_id"] == job["session_id"]
assert manifest["artifacts"] == ["安卓截图.png"]
assert archive.read("1.1/任务文件/安卓/安卓截图.png") == b"native png"
assert not any("partial" in name for name in files)
assert "1.1-all.zip" in response.headers["content-disposition"]
def test_batch_zip_contains_each_tasks_android_artifacts(isolated):
batch = service.create_batch(payload())
for index, job in enumerate(isolated.jobs(batch["id"])):
root = reports.TRACES_PATH / job["session_id"] / "artifacts" / "android"
root.mkdir(parents=True)
(root / "结果.txt").write_text(f"任务{index}", encoding="utf-8")
with zipfile.ZipFile(reports.export_batch(batch["id"])) as archive:
names = [name for name in archive.namelist() if name.endswith("任务文件/安卓/结果.txt")]
assert len(names) == 2
assert {archive.read(name).decode() for name in names} == {"任务0", "任务1"}
def test_android_artifact_export_does_not_follow_other_session_links(isolated):
root = reports.TRACES_PATH / "session" / "artifacts"
root.mkdir(parents=True)
other = reports.TRACES_PATH / "other-session"
other.mkdir()
(other / "private.txt").write_text("other task")
try:
(root / "android").symlink_to(other, target_is_directory=True)
except OSError:
pytest.skip("symlinks unavailable")
assert reports.session_artifacts({"id": "session"}) == []
def test_video_export_accepts_finalized_worker_trace_and_rejects_external_path(
isolated, tmp_path, monkeypatch
):
+1 -1
View File
@@ -151,7 +151,7 @@ def test_full_set_enum_slots_render_verbatim():
" `press_key`, `manage_app`, or `wait_for_delay`)" in out
)
assert "Helper/Subagent tools (`ask_explorer`, `ask_diagnoser`, `video_analyzer`)" in out
assert "ADB/task tools (`run_adb_command`, `manage_task`)" in out
assert "ADB/task tools (`run_adb_command`, `manage_task`, `android_file`)" in out
assert (
"(`read_note`, `list_notes`, `search_history`, `replay_steps`, `get_step_screenshot`)"
in out
@@ -197,6 +197,15 @@ async def test_perception_node_async_offloading(mock_artemis_ctx, tmp_path):
engine = DataEngine(mock_artemis_ctx)
engine.start_session("Test Perception Node Offloading")
mock_artemis_ctx.data_engine = engine
image_saved = asyncio.Event()
loop = asyncio.get_running_loop()
save_image = engine.get_or_create_image
def save_image_and_signal(*args, **kwargs):
try:
return save_image(*args, **kwargs)
finally:
loop.call_soon_threadsafe(image_saved.set)
# Create an injected_instruction.json file
instruction_file = Path(engine.base_dir) / "injected_instruction.json"
@@ -214,6 +223,7 @@ async def test_perception_node_async_offloading(mock_artemis_ctx, tmp_path):
mock_screen_data.height = 2400
with (
patch.object(engine, "get_or_create_image", side_effect=save_image_and_signal),
patch("artemis.graph.perception.UnifiedMobileController") as mock_controller_cls,
patch("artemis.graph.perception.perform_ocr", new_callable=AsyncMock) as mock_ocr,
patch("artemis.graph.perception._detect_status_bar_height", return_value=0),
@@ -240,10 +250,9 @@ async def test_perception_node_async_offloading(mock_artemis_ctx, tmp_path):
# Verify injected_instruction.json was unlinked
assert not instruction_file.exists()
# Await background tasks triggered by perception_node
await asyncio.sleep(0.1)
if engine._pending_tasks:
await asyncio.gather(*engine._pending_tasks)
# perception_node creates its own task, outside engine._pending_tasks.
# Wait for actual persistence instead of assuming the disk finishes in 100 ms.
await asyncio.wait_for(image_saved.wait(), timeout=5)
# Verify image record and file were created in background by perception_node
img_record = engine.storage.get_image(expected_hash)
+25 -8
View File
@@ -7,6 +7,7 @@ import logging
import os
from pathlib import Path
import threading
from types import SimpleNamespace
import pytest
@@ -35,8 +36,12 @@ def test_running_state_recovers_from_temporary_windows_file_access(tmp_path, mon
raise _windows_access_error(winerror)
return replace(source, target)
monkeypatch.setattr(task_control.os, "replace", busy_replace)
monkeypatch.setattr(task_control.time, "sleep", delays.append)
monkeypatch.setattr(
task_control, "os", SimpleNamespace(**{**vars(task_control.os), "replace": busy_replace})
)
monkeypatch.setattr(
task_control, "time", SimpleNamespace(**{**vars(task_control.time), "sleep": delays.append})
)
task_control.TaskControl("task-123", object())._state("running")
assert task_control.read_state("task-123")["status"] == "running"
@@ -55,8 +60,12 @@ def test_persistent_windows_access_error_is_bounded_and_preserves_old_state(tmp_
attempts.append(source)
raise error
monkeypatch.setattr(task_control.os, "replace", denied_replace)
monkeypatch.setattr(task_control.time, "sleep", delays.append)
monkeypatch.setattr(
task_control, "os", SimpleNamespace(**{**vars(task_control.os), "replace": denied_replace})
)
monkeypatch.setattr(
task_control, "time", SimpleNamespace(**{**vars(task_control.time), "sleep": delays.append})
)
with pytest.raises(PermissionError) as raised:
task_control._write_atomic(path, {"status": "running"})
@@ -82,8 +91,12 @@ def test_unrelated_file_errors_fail_without_retry_and_clean_temporary_file(
attempts.append(source)
raise error
monkeypatch.setattr(task_control.os, "replace", failed_replace)
monkeypatch.setattr(task_control.time, "sleep", delays.append)
monkeypatch.setattr(
task_control, "os", SimpleNamespace(**{**vars(task_control.os), "replace": failed_replace})
)
monkeypatch.setattr(
task_control, "time", SimpleNamespace(**{**vars(task_control.time), "sleep": delays.append})
)
with pytest.raises(OSError) as raised:
task_control._write_atomic(path, {"status": "running"})
@@ -127,7 +140,9 @@ def test_busy_temporary_cleanup_preserves_original_write_error(tmp_path, monkeyp
raise _windows_access_error(32)
return unlink(target, **kwargs)
monkeypatch.setattr(task_control.os, "replace", failed_replace)
monkeypatch.setattr(
task_control, "os", SimpleNamespace(**{**vars(task_control.os), "replace": failed_replace})
)
monkeypatch.setattr(Path, "unlink", busy_unlink)
monkeypatch.setattr(task_control.time, "sleep", lambda _: None)
try:
@@ -163,7 +178,9 @@ def test_repeated_control_requests_recover_from_file_sharing(
raise _windows_access_error(32)
return replace(source, target)
monkeypatch.setattr(task_control.os, "replace", busy_replace)
monkeypatch.setattr(
task_control, "os", SimpleNamespace(**{**vars(task_control.os), "replace": busy_replace})
)
monkeypatch.setattr(task_control.time, "sleep", lambda _: None)
send_request("task-123")
+307
View File
@@ -0,0 +1,307 @@
"""Android artifact integrity, path separation and Flash/Pro tool parity."""
import asyncio
import hashlib
from io import BytesIO
import json
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock
from PIL import Image
import pytest
from artemis.agents.flash.runner import FlashRunner
from artemis.agents.validator.tool_declarations import ToolExecutionResult
from artemis.core.tool_failure import is_tool_failure
from artemis.mcp.action_executor import McpActionExecutor
from artemis.tools import android_files as files
@pytest.fixture
def ctx(tmp_path):
return SimpleNamespace(
execution_target="mixed",
device=SimpleNamespace(device_id="task-device"),
data_engine=SimpleNamespace(base_dir=tmp_path / "trace", current_session_id="session"),
execution_setup=None,
actuator=None,
plugins=None,
)
@pytest.fixture
def png():
stream = BytesIO()
Image.new("RGB", (144, 312), "red").save(stream, "PNG")
return stream.getvalue()
@pytest.mark.asyncio
async def test_screenshot_saved_native_and_desktop_copy_verified(ctx, png, tmp_path, monkeypatch):
desktop = tmp_path / "重定向桌面"
desktop.mkdir()
monkeypatch.setattr(files, "desktop_directory", lambda: desktop)
run = AsyncMock(return_value=png)
monkeypatch.setattr(files.AndroidFiles, "_run", run)
result = json.loads(
await files.android_file.execute(
ctx=ctx, action="screenshot", filename="手机截图.png", destination="desktop"
)
)
assert result["ok"] and result["verified"]
assert result["width"] == 144 and result["height"] == 312
assert result["device_serial"] == "task-device"
assert Path(result["path"]) == desktop / "手机截图.png"
assert Path(result["path"]).read_bytes() == Path(result["trace_path"]).read_bytes() == png
assert result["sha256"] == hashlib.sha256(png).hexdigest()
run.assert_awaited_once_with("exec-out", "screencap", "-p", limit=files.MAX_FILE_BYTES)
@pytest.mark.asyncio
async def test_existing_artifact_is_preserved_unless_explicit_overwrite(ctx, png, monkeypatch):
monkeypatch.setattr(files.AndroidFiles, "_run", AsyncMock(return_value=png))
first = json.loads(
await files.android_file.execute(ctx=ctx, action="screenshot", filename="a.png")
)
path = Path(first["trace_path"])
path.write_bytes(b"original")
error = await files.android_file.execute(ctx=ctx, action="screenshot", filename="a.png")
assert is_tool_failure(error) and path.read_bytes() == b"original"
result = await files.android_file.execute(
ctx=ctx, action="screenshot", filename="a.png", overwrite=True
)
assert json.loads(result)["ok"] and path.read_bytes() == png
@pytest.mark.asyncio
@pytest.mark.parametrize(
"name", ["../escape.png", "C:\\out.png", "x:y.png", "CON.png", "x.png ", "a\n.png"]
)
async def test_invalid_host_paths_fail_before_adb(ctx, name, monkeypatch):
run = AsyncMock()
monkeypatch.setattr(files.AndroidFiles, "_run", run)
result = await files.android_file.execute(ctx=ctx, action="screenshot", filename=name)
assert is_tool_failure(result)
run.assert_not_awaited()
@pytest.mark.asyncio
@pytest.mark.parametrize(
"device_path", ["C:\\Desktop\\x", "relative", "/sdcard/../x", "/", "/sdcard/a\n"]
)
async def test_android_path_validation(ctx, device_path, monkeypatch):
run = AsyncMock()
monkeypatch.setattr(files.AndroidFiles, "_run", run)
assert is_tool_failure(
await files.android_file.execute(ctx=ctx, action="read_text", device_path=device_path)
)
run.assert_not_awaited()
@pytest.mark.asyncio
async def test_invalid_image_does_not_create_artifact(ctx, monkeypatch):
monkeypatch.setattr(files.AndroidFiles, "_run", AsyncMock(return_value=b"screencap failed"))
assert is_tool_failure(await files.android_file.execute(ctx=ctx, action="screenshot"))
assert not (ctx.data_engine.base_dir / "artifacts").exists()
@pytest.mark.asyncio
async def test_read_and_pull_verify_chinese_and_binary_data(ctx, monkeypatch):
service = files.AndroidFiles(ctx)
data = "中文内容\n第二行".encode()
digest = hashlib.sha256(data).hexdigest()
shell = AsyncMock(side_effect=[str(len(data)), digest, digest])
monkeypatch.setattr(service, "_shell", shell)
monkeypatch.setattr(service, "_run", AsyncMock(return_value=data))
result = await service.execute(
files.AndroidFileArgs(action="read_text", device_path="/sdcard/a '中'.txt")
)
assert result["content"] == data.decode()
assert "'\"'\"'" in shell.call_args_list[0].args[0] # POSIX shell quoting
binary = b"\x00\xff\r\n\x1a"
digest = hashlib.sha256(binary).hexdigest()
shell.side_effect = [str(len(binary)), digest, digest]
service._run.return_value = binary
result = await service.execute(
files.AndroidFileArgs(action="pull", device_path="/sdcard/file.bin")
)
assert Path(result["path"]).read_bytes() == binary
@pytest.mark.asyncio
async def test_changed_or_oversize_file_is_not_saved(ctx, monkeypatch):
service = files.AndroidFiles(ctx)
monkeypatch.setattr(service, "_shell", AsyncMock(side_effect=["3", "0" * 64]))
run = AsyncMock(return_value=b"abc")
monkeypatch.setattr(service, "_run", run)
with pytest.raises(OSError, match="传输"):
await service.execute(files.AndroidFileArgs(action="pull", device_path="/sdcard/a"))
assert not service.root.exists()
service._shell.side_effect = [str(files.MAX_TEXT_BYTES + 1)]
run.reset_mock()
with pytest.raises(ValueError, match="上限"):
await service.execute(files.AndroidFileArgs(action="read_text", device_path="/sdcard/a"))
run.assert_not_awaited()
@pytest.mark.asyncio
async def test_write_text_pushes_utf8_and_verifies_before_rename(ctx, monkeypatch):
service = files.AndroidFiles(ctx)
data = "第一行\n第二行".encode()
calls = []
async def run(*args, **kwargs):
if args[0] == "push":
assert Path(args[1]).read_bytes() == data
calls.append("push")
return b""
async def shell(command):
calls.append(command)
return hashlib.sha256(data).hexdigest() if command.startswith("sha256sum") else ""
monkeypatch.setattr(service, "_run", run)
monkeypatch.setattr(service, "_shell", shell)
result = await service.execute(
files.AndroidFileArgs(
action="write_text", device_path="/sdcard/Download/文本.txt", content=data.decode()
)
)
assert result["verified"] and result["bytes"] == len(data)
assert any("mv -n" in call and "test ! -e" in call for call in calls)
assert calls.index("push") < next(i for i, c in enumerate(calls) if "mv -n" in c)
assert not list(service.root.iterdir()) # no leaked local staging files
@pytest.mark.asyncio
async def test_permission_failure_propagates_and_does_not_claim_success(ctx, monkeypatch):
monkeypatch.setattr(
files.AndroidFiles, "_run", AsyncMock(side_effect=OSError("Permission denied"))
)
result = await files.android_file.execute(
ctx=ctx, action="read_text", device_path="/data/private.txt"
)
assert is_tool_failure(result) and "Permission denied" in result
@pytest.mark.asyncio
async def test_existing_android_file_is_never_pushed_over(ctx, monkeypatch):
service = files.AndroidFiles(ctx)
monkeypatch.setattr(service, "_shell", AsyncMock(return_value="exists"))
push = AsyncMock()
monkeypatch.setattr(service, "_run", push)
with pytest.raises(FileExistsError, match="设备文件已存在"):
await service.execute(
files.AndroidFileArgs(action="write_text", device_path="/sdcard/a", content="new")
)
push.assert_not_awaited()
@pytest.mark.asyncio
@pytest.mark.parametrize("failure", ["cancel", "timeout", "limit"])
async def test_adb_subprocess_is_reaped_on_interrupt(ctx, monkeypatch, failure):
service = files.AndroidFiles(ctx)
process = Mock(returncode=None)
process.stdout.read = AsyncMock()
process.stderr.read = AsyncMock(return_value=b"")
if failure == "limit":
process.stdout.read.return_value = b"too large"
expected = ValueError
else:
async def wait_forever(*args):
await asyncio.Event().wait()
process.stdout.read.side_effect = wait_forever
expected = asyncio.CancelledError if failure == "cancel" else TimeoutError
process.wait = AsyncMock(return_value=0)
def kill():
process.returncode = -1
process.kill.side_effect = kill
spawn = AsyncMock(return_value=process)
monkeypatch.setattr(asyncio, "create_subprocess_exec", spawn)
monkeypatch.setattr(files, "ADB_TIMEOUT_SECONDS", 0.02)
task = asyncio.create_task(service._run("exec-out", "cat /sdcard/a", limit=2))
await asyncio.sleep(0)
if failure == "cancel":
task.cancel()
with pytest.raises(expected):
await task
process.kill.assert_called_once()
process.wait.assert_awaited()
argv = spawn.call_args.args
assert argv[argv.index("-s") + 1] == "task-device"
assert "-H" in argv and "-P" in argv
@pytest.mark.asyncio
async def test_flash_and_pro_share_schema_handler_and_failure_status(ctx, png, monkeypatch):
monkeypatch.setattr(files.AndroidFiles, "_run", AsyncMock(return_value=png))
monkeypatch.setattr("artemis.tools.base.get_driver", lambda ctx: None)
monkeypatch.setattr(
"artemis.mcp.action_executor.TraceSpan",
lambda **kw: Mock(__enter__=lambda self: self, __exit__=lambda *a: False),
)
runner = object.__new__(FlashRunner)
runner.ctx = ctx
assert "android_file" in {t.name for t in runner._get_tools()}
pro = files.android_file.to_langchain_tool(ctx)
assert pro.args_schema is files.AndroidFileArgs
result = await pro.ainvoke({"action": "screenshot", "filename": "pro.png"})
assert json.loads(result)["verified"]
executor = McpActionExecutor(ctx, actuator=Mock())
result = await executor.execute(
"android_file", {"action": "screenshot", "filename": "flash.png"}, "call", None
)
assert result.status == "success" and json.loads(result.text_summary)["verified"]
result = await executor.execute(
"android_file", {"action": "screenshot", "filename": "flash.png"}, "call2", None
)
assert result.status == "error"
ctx.execution_target = "computer"
assert "android_file" not in {t.name for t in runner._get_tools()}
assert is_tool_failure(await files.android_file.execute(ctx=ctx, action="screenshot"))
@pytest.mark.asyncio
@pytest.mark.parametrize("status", ["success", "error"])
async def test_flash_records_file_receipts_as_steps_for_final_verification(ctx, status):
runner = object.__new__(FlashRunner)
runner.ctx = ctx
runner._llm = None
runner.summarizer = None
result = ToolExecutionResult(
tool_call_id="file-call",
tool_name="android_file",
status=status,
text_summary='{"verified": true, "sha256": "receipt"}'
if status == "success"
else "Permission denied",
)
runner.executor = SimpleNamespace(
action_tool_names=frozenset({"click"}), execute=AsyncMock(return_value=result)
)
runner._record_action_step = Mock(return_value="recorded-file-step")
turn = SimpleNamespace(step_keys=[], actions=[])
messages = []
await runner._execute_and_record_action(
name="android_file",
args={"action": "screenshot"},
tc_id="file-call",
state=SimpleNamespace(),
messages=messages,
raw_text="保存当前安卓屏幕",
step_token_usage={},
pre_screenshot_bytes=b"previous-observation",
xml_list="UI",
action_sequence=0,
turn=turn,
)
runner._record_action_step.assert_called_once()
assert runner._record_action_step.call_args.args[2] is result
assert turn.step_keys == ["recorded-file-step"]
assert turn.actions == [("android_file", status, result.text_summary)]
assert messages[-1].content == result.text_summary
+72
View File
@@ -136,6 +136,78 @@ command = "leave-alone"
assert path.with_name(path.name + ".artemis-update.bak").read_text(encoding="utf-8") == original
@pytest.mark.parametrize(
"policies",
[
'# Keep my tool policies\ntools = { mobile_run_task = { enabled = false, approval_policy = "on-request" }, retired_tool = { enabled = false } }\n',
'# Keep my tool policies\ntools.mobile_run_task.enabled = false\ntools.mobile_run_task.approval_policy = "on-request"\ntools.retired_tool.enabled = false\n',
'# Keep my tool policies\n[mcp_servers.artemis.tools.mobile_run_task]\nenabled = false\napproval_policy = "on-request"\n[mcp_servers.artemis.tools.retired_tool]\nenabled = false\n',
],
ids=["inline-table", "dotted-keys", "tool-subtables"],
)
def test_codex_sync_preserves_tool_policies_verbatim(tmp_path, monkeypatch, policies):
from artemis.interfaces.cli.commands import mcp as mcp_command
codex_home = tmp_path / "中文用户" / ".codex"
codex_home.mkdir(parents=True)
path = codex_home / "config.toml"
original = (
'[mcp_servers.artemis]\ncommand = "old-python"\n'
'args = ["-m", "mcp_server"]\n'
'enabled_tools = ["mobile_run_task"]\n'
'disabled_tools = ["mobile_config"]\n'
'env = { PYTHONPATH = "C:/old", OPENAI_API_KEY = "test-secret" }\n'
+ policies
+ '[mcp_servers.other]\ncommand = "leave-alone"\n'
)
path.write_text(original, encoding="utf-8")
before = tomllib.loads(original)
monkeypatch.setenv("CODEX_HOME", str(codex_home))
root = tmp_path / "新版项目"
assert update.refresh_mcp(root) == 0
refreshed = path.read_text(encoding="utf-8")
after = tomllib.loads(refreshed)
entry = after["mcp_servers"]["artemis"]
assert entry["command"] == update.sys.executable
assert entry["cwd"] == str(root)
assert entry["env"] == {
"PYTHONPATH": str(root),
"OPENAI_API_KEY": "test-secret",
"PYTHONUNBUFFERED": "1",
"ARTEMIS_DESKTOP_NOTIFY": "true",
}
assert entry["enabled_tools"] == list(mcp_command.ARTEMIS_MCP_TOOLS)
assert entry["disabled_tools"] == ["mobile_config"]
assert entry["tools"] == before["mcp_servers"]["artemis"]["tools"]
assert policies.split("\n", 1)[1] in refreshed
assert refreshed.count("# Keep my tool policies") == 1
assert after["mcp_servers"]["other"] == before["mcp_servers"]["other"]
assert path.with_name(path.name + ".artemis-update.bak").read_text(encoding="utf-8") == original
assert update.refresh_mcp(root) == 0
assert path.read_text(encoding="utf-8") == refreshed
def test_codex_sync_reports_generated_toml_error_without_blame_or_overwrite(
tmp_path, monkeypatch, capsys
):
codex_home = tmp_path / ".codex"
codex_home.mkdir()
path = codex_home / "config.toml"
original = '[mcp_servers.artemis]\ncommand = "old"\nargs = ["-m", "mcp_server"]\n'
path.write_text(original, encoding="utf-8")
monkeypatch.setenv("CODEX_HOME", str(codex_home))
monkeypatch.setenv("ARTEMIS_LAUNCH_LANG", "zh-CN")
monkeypatch.setattr(update, "_replace_toml_tables", lambda *_: 'tools = {"secret": true}\n')
assert update.refresh_mcp(tmp_path / "project") == 1
assert path.read_text(encoding="utf-8") == original
assert not path.with_name(path.name + ".artemis-update.bak").exists()
output = capsys.readouterr().out
assert "原配置可正常解析" in output
assert "ARTEMIS 同步错误" in output
assert "配置结构无效" not in output
assert "secret" not in output
def test_mcp_sync_updates_only_codex_and_preserves_other_clients(tmp_path, monkeypatch):
from artemis.interfaces.cli.commands import mcp as mcp_command