feat: add Android task file evidence and finalize plugin design

This commit is contained in:
chenmingxuan committed 2026-10-15 18:27:11 +08:00
1 parent 96fedc816f
commit 72ff342605
20 files changed
+1005 -35

No files matched your search

@@ -792,6 +792,22 @@ def session_videos(report: dict) -> list[tuple[Path, str]]:
return [(path, f"{i:04d}_{path.parent.name}_{path.name}") for i, path in enumerate(paths, 1)]
def session_artifacts(report: dict) -> list[tuple[Path, str]]:
"""Only completed, task-owned Android files; never follow external links."""
traces = TRACES_PATH.resolve()
root = traces / identifier(report["id"]) / "artifacts" / "android"
if not root.is_dir() or root.resolve() != root:
return []
return [
(path, path.name)
for path in sorted(root.iterdir())
if path.is_file()
and not path.is_symlink()
and not path.name.startswith(".artemis-")
and path.resolve().parent == root.resolve()
]
def export_session_media(session_id: str, kind: str) -> Path:
if kind not in {"screenshots", "video", "all"}:
raise ValueError("不支持的下载类型")
@@ -801,6 +817,7 @@ def export_session_media(session_id: str, kind: str) -> Path:
target_dir.mkdir(parents=True, exist_ok=True)
images = session_screenshots(report) if kind in {"screenshots", "all"} else []
videos = session_videos(report) if kind in {"video", "all"} else []
artifacts = session_artifacts(report) if kind == "all" else []
if kind == "video" and not videos:
raise ValueError("此任务没有已保存的录屏")
if kind == "screenshots" and not images:
@@ -814,6 +831,7 @@ def export_session_media(session_id: str, kind: str) -> Path:
"verdict": report["verdict"],
"screenshots": [name for _, name in images],
"videos": [name for _, name in videos],
"artifacts": [name for _, name in artifacts],
"note": "录屏按时间顺序分段;暂停期间未占用设备,也未录制其他任务。",
}
prefix = archive_case_name(report) + "/" if kind == "all" else ""
@@ -825,6 +843,8 @@ def export_session_media(session_id: str, kind: str) -> Path:
archive.write(source, f"{prefix}步骤截图/{name}")
for source, name in videos:
archive.write(source, f"{prefix}任务录屏/{name}")
for source, name in artifacts:
archive.write(source, f"{prefix}任务文件/安卓/{name}")
archive.writestr(
prefix + "文件清单.json", json.dumps(manifest, ensure_ascii=False, indent=2)
)
@@ -882,6 +902,8 @@ def export_batch(batch_id: str, progress: Callable[[int, int, str], None] | None
archive.write(source, f"{prefix}/步骤截图/{name}")
for source, name in session_videos(report):
archive.write(source, f"{prefix}/任务录屏/{name}")
for source, name in session_artifacts(report):
archive.write(source, f"{prefix}/任务文件/安卓/{name}")
if progress:
progress(number, len(records), f"已打包任务 {number}/{len(records)}")
if progress:
+1
View File
@@ -31,6 +31,7 @@ Keep in mind that the mobile device is an asynchronous, constantly evolving envi
---
# 3. READING YOUR HISTORY & THE SESSION CLOCK
{% if "android_file" in available_tools %}- **Android files (`android_file`)**: To save the current phone screenshot, call `action="screenshot"`; do not use PC screenshot shortcuts or the computer plugin. Files default to this task's trace, with an additional desktop copy only for `destination="desktop"`. The list/read_text/write_text/pull/push actions operate on the selected Android device; device_path is an Android path, never a Windows path. Check the returned verified path/hash before claiming a file was saved. Saving a screenshot does not verify that its page meets the goal; inspect the screen too. Use a new filename unless the user explicitly authorized overwriting.{% endif %}
- **Conversation = execution history**: The earlier turns of this conversation are your raw execution history. Each observation opens with `# CURRENT OBSERVATION [T+mm:ss]`. Every device action runs immediately and its tool result states what the device did (e.g. `Swiped up.`); whether it had the intended effect is yours to judge from the observation that follows. Only a turn whose action failed ends with an `--- Action Execution Result (T+mm:ss) ---` message (`Status: failed` with the `Error:` line). Every timestamp is a session-relative `T+mm:ss` offset from the start of this session; it is the only clock you have, for pacing waits as well as for reasoning about elapsed time.
- **Compression**: Screenshots older than a few turns are replaced by short `--- Historical Visual Transition ---` summaries and old `--- Visible UI Elements ---` lists are dropped. Long spans render as `[Era N | Steps a–b | T+.. → T+..]` blocks (a merged synopsis, segment titles, and a per-step action ledger). Because of this, if you spot critical text that you will need later (e.g., verification codes, specific titles, search lists), **you must explicitly verbalize and write those details into your text reasoning** so they persist across turns.
{% if "search_history" in available_tools %}- **History Lookup (`search_history`)**: Compressed history keeps the step numbers and `T+` offsets but not every detail. When you need exactly what you did or saw at an earlier step (a value you read, the result of an action, which screen you were on), call `search_history` with keywords and/or the `[start, end]` step range shown in a chunk block; it searches the full stored history (screen descriptions, actions, results, reasoning, tool calls, notes, on-screen text) and every hit carries its step number. It is a pre-decision helper: read its answer, then act.{% endif %}
+13 -3
View File
@@ -91,6 +91,7 @@ from artemis.services.llm import (
invoke_llm_with_timeout_message,
)
from artemis.tools.history import history_tool_declarations
from artemis.tools.android_files import android_file
from artemis.tools.tool_wrapper import tool_result_messages
from artemis.utils.coordinates import (
COORDINATE_SPACE_KEY,
@@ -212,6 +213,8 @@ class FlashRunner:
# declared from the very same args schemas the LangChain tools bind
# (same availability gates: a DataEngine session, recall config).
tools.extend(history_tool_declarations(self.ctx))
if android_file.is_available(self.ctx):
tools.append(android_file.to_tool_declaration())
if self._video_tools_enabled():
from artemis.tools.video_tool import VIDEO_ANALYZER_TOOL
@@ -631,7 +634,9 @@ class FlashRunner:
and getattr(self.ctx, "execution_target", "android") != "computer"
):
return post_img_bytes
if post_img_bytes or name not in action_names:
# File operations have their own verified receipts. Do not add a UI
# observation/visual summary that could obscure that non-visual evidence.
if post_img_bytes or name not in action_names or name == "android_file":
return post_img_bytes
if state is not None:
state.indexed_points = []
@@ -733,6 +738,9 @@ class FlashRunner:
)
action_dict.pop("coordinates", None)
action_dict.pop(COORDINATE_SPACE_KEY, None)
if name == "android_file":
action_dict.pop("coordinates", None)
action_dict.pop(COORDINATE_SPACE_KEY, None)
return action_dict
def _record_action_step(
@@ -827,7 +835,9 @@ class FlashRunner:
"""
# Dynamic dispatch set: manifest device actions plus any backend
# extension tools, so extension steps are recorded like actions.
action_names = self.executor.action_tool_names
# File helpers perform real work too. Persist their receipts as steps,
# otherwise helper-only turns vanish from the final checker's evidence.
action_names = self.executor.action_tool_names | {"android_file"}
plugin_action = False
try:
operation = self.executor.execute(
@@ -1017,7 +1027,7 @@ class FlashRunner:
Returns (final_report, pre_screenshot_bytes, xml_list, action_sequence);
final_report is non-None only when 'report_task_status' was called.
"""
action_names = self.executor.action_tool_names
action_names = self.executor.action_tool_names | {"android_file"}
index_elements = self._turn_index_snapshot(state)
for tc in tool_calls:
control = getattr(self.ctx, "task_control", None)
+5 -1
View File
@@ -64,7 +64,7 @@ def load_operator_prompts() -> dict[str, str]:
# artemis/agents/prompt_assembly.py for the assembly rationale.
_PRE_DECISION_HELPER_TOOLS = ("ask_explorer", "ask_diagnoser", "video_analyzer")
_PRE_DECISION_ADB_TOOLS = ("run_adb_command", "manage_task")
_PRE_DECISION_ADB_TOOLS = ("run_adb_command", "manage_task", "android_file")
_PRE_DECISION_MEMORY_TOOLS = (
"read_note",
"list_notes",
@@ -203,6 +203,10 @@ def resolve_operator_prompt_tools(ctx: ArtemisContext) -> frozenset[str]:
# video_recording_tools_enabled); the prompt must not advertise it when the
# tool is not actually available this run.
available = set(OPERATOR_PROMPT_TOOLSET)
from artemis.tools.android_files import android_file
if not android_file.is_available(ctx):
available.discard("android_file")
setup = getattr(ctx, "execution_setup", None)
if not (setup and getattr(setup, "video_recording_tools_enabled", False)):
available.discard("video_analyzer")
+5
View File
@@ -1051,6 +1051,11 @@ async def get_graph(ctx: ArtemisContext) -> CompiledStateGraph:
operator_tools = operator_native_tools + operator_specialized_tools
from artemis.tools.android_files import android_file
if android_file.is_available(ctx):
operator_tools.append(android_file.to_langchain_tool(ctx))
if ctx.execution_setup and ctx.execution_setup.video_recording_tools_enabled:
operator_tools.append(get_video_analyzer_tool(ctx, role="operator"))
+5 -1
View File
@@ -51,6 +51,7 @@ from artemis.agents.validator.tool_declarations import (
normalize_coordinate_target,
)
from artemis.tools.history import HISTORY_TOOL_NAMES, history_tool_by_name
from artemis.tools.android_files import android_file
from artemis.tools.tool_wrapper import split_multimodal_result
from artemis.utils.coordinates import (
compute_smart_swipe_coordinates,
@@ -73,7 +74,8 @@ __all__ = ["McpActionExecutor"]
#: Non-device tools executed agent-side (never behind the action server).
AGENT_TOOL_NAMES: frozenset[str] = (
frozenset({"read_note", "list_notes", "ask_explorer", "video_analyzer"}) | HISTORY_TOOL_NAMES
frozenset({"read_note", "list_notes", "ask_explorer", "video_analyzer", "android_file"})
| HISTORY_TOOL_NAMES
)
#: Actions whose ``target`` is a single point: an element index or an [x, y] pair.
@@ -769,6 +771,8 @@ class McpActionExecutor:
)
elif raw_name in HISTORY_TOOL_NAMES:
text, blocks = await self._history_tool(raw_name, args)
elif raw_name == "android_file":
text = await android_file.execute(ctx=self.ctx, **args)
else:
text, ok = await self._ask_explorer(
# ``task_description`` is the pre-contract argument name
+1
View File
@@ -122,6 +122,7 @@ BACKEND_INDEPENDENT_TOOLS: frozenset[str] = frozenset(
"ask_committee",
"video_analyzer",
"run_adb_command",
"android_file",
"manage_task",
"analyze_task_output",
"search_history",
+348
View File
@@ -0,0 +1,348 @@
"""Task-scoped Android files and native-resolution screenshot artifacts."""
import asyncio
import ctypes
from datetime import datetime, UTC
import hashlib
from io import BytesIO
import json
import os
from pathlib import Path, PurePosixPath
import re
import shlex
from typing import Any, Literal
import uuid
from PIL import Image
from pydantic import BaseModel, Field
from artemis.core.tool_failure import ToolFailure
from artemis.runtime.adb_endpoint import AdbSession, current_adb_endpoint
from artemis.tools.base import ArtemisTool
# Bound memory, model context and subprocess lifetime independently of task timeout.
MAX_FILE_BYTES = 64 * 1024 * 1024
MAX_TEXT_BYTES = 64 * 1024
ADB_TIMEOUT_SECONDS = 60
def android_files_available(ctx: Any) -> bool:
engine = getattr(ctx, "data_engine", None)
return bool(
getattr(ctx, "execution_target", "android") != "computer"
and getattr(getattr(ctx, "device", None), "device_id", None)
and engine is not None
and getattr(engine, "current_session_id", None)
)
def desktop_directory() -> Path:
"""Resolve the actual Windows desktop, including redirected/OneDrive desktops."""
if os.name == "nt":
buffer = ctypes.create_unicode_buffer(32768)
if ctypes.windll.shell32.SHGetFolderPathW(None, 0x10, None, 0, buffer) != 0:
raise OSError("无法定位当前用户的桌面目录")
path = Path(buffer.value)
else:
path = Path.home() / "Desktop"
if not path.is_dir():
raise OSError("桌面目录不存在;请使用 destination=trace")
return path.resolve()
def _filename(value: str) -> str:
if (
not value
or len(value) > 180
or value in {".", ".."}
or value != value.strip(" .")
or re.search(r'[\\/:*?"<>|\x00-\x1f]', value)
or re.match(r"^(CON|PRN|AUX|NUL|COM[0-9]|LPT[0-9])(?:\.|$)", value, re.I)
or value.startswith(".artemis-")
):
raise ValueError("filename 必须是普通文件名,不能包含路径或 Windows 保留名称")
return value
def _device_path(value: str) -> str:
path = PurePosixPath(value)
if (
not value.startswith("/")
or value.startswith("//")
or path == PurePosixPath("/")
or ".." in path.parts
or any(ord(char) < 32 for char in value)
):
raise ValueError("device_path 必须是 Android 绝对路径,不能包含 .. 或控制字符")
return str(path)
def _host_path(root: Path, name: str) -> Path:
path = root / _filename(name)
if path.is_symlink() or path.resolve().parent != root.resolve():
raise ValueError("文件不能是符号链接或指向目标目录之外")
return path
class AndroidFiles:
"""Use only the task's selected serial and its process-pinned ADB endpoint."""
def __init__(self, ctx: Any):
if not android_files_available(ctx):
raise ValueError("安卓文件工具需要当前任务的 Android 设备和 trace 会话")
self.serial = str(ctx.device.device_id)
self.root = Path(ctx.data_engine.base_dir) / "artifacts" / "android"
self.adb = AdbSession(current_adb_endpoint())
async def _run(self, *args: str, limit: int = MAX_TEXT_BYTES) -> bytes:
process = await asyncio.create_subprocess_exec(
*self.adb.command(["-s", self.serial, *args]),
env=self.adb.environment(),
stdin=asyncio.subprocess.DEVNULL,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
**({"creationflags": 0x08000000} if os.name == "nt" else {}),
)
async def bounded_read(stream, maximum):
output = bytearray()
while chunk := await stream.read(65536):
output.extend(chunk)
if len(output) > maximum:
raise ValueError(f"ADB 输出超过 {maximum} 字节限制")
return bytes(output)
readers = [
asyncio.create_task(bounded_read(process.stdout, limit)),
asyncio.create_task(bounded_read(process.stderr, MAX_TEXT_BYTES)),
]
try:
async with asyncio.timeout(ADB_TIMEOUT_SECONDS):
stdout, stderr = await asyncio.gather(*readers)
await process.wait()
if process.returncode:
detail = (stderr or stdout).decode("utf-8", errors="replace")[:2000]
raise OSError(f"ADB 文件操作失败({process.returncode}):{detail}")
return stdout
finally:
if process.returncode is None:
try:
process.kill()
except ProcessLookupError:
pass
await process.wait()
for reader in readers:
if not reader.done():
reader.cancel()
await asyncio.gather(*readers, return_exceptions=True)
async def _shell(self, command: str) -> str:
return (await self._run("shell", command)).decode("utf-8", errors="strict").strip()
async def _hash(self, path: str) -> str:
result = await self._shell(f"sha256sum {shlex.quote(path)}")
digest = result.split(maxsplit=1)[0] if result else ""
if not re.fullmatch(r"[0-9a-fA-F]{64}", digest):
raise OSError("设备未返回有效 SHA-256,无法确认文件内容")
return digest.lower()
async def _read(self, path: str, limit: int) -> bytes:
quoted = shlex.quote(path)
size = int(await self._shell(f"test -f {quoted} && stat -c %s {quoted}"))
if not 0 <= size <= limit:
raise ValueError(f"文件大小 {size} 超过上限 {limit} 字节,请缩小文件或使用 pull")
before = await self._hash(path)
data = await self._run("exec-out", f"cat {quoted}", limit=limit)
if len(data) != size or hashlib.sha256(data).hexdigest() != before:
raise OSError("文件传输不完整或传输期间内容发生变化,未保存")
if await self._hash(path) != before:
raise OSError("传输期间设备文件发生变化,未保存")
return data
def _save(self, data: bytes, name: str, destination: str, overwrite: bool) -> dict:
self.root.mkdir(parents=True, exist_ok=True)
trace_path = _host_path(self.root, name)
targets = [trace_path]
if destination == "desktop":
targets.append(_host_path(desktop_directory(), name))
for path in targets:
if path.exists() and not overwrite:
raise FileExistsError(f"文件已存在:{path};换名或在获准覆盖后使用 overwrite=true")
digest = hashlib.sha256(data).hexdigest()
for path in targets:
temporary = path.parent / f".artemis-{uuid.uuid4().hex}.tmp"
try:
with temporary.open("xb") as handle:
handle.write(data)
handle.flush()
os.fsync(handle.fileno())
if hashlib.sha256(temporary.read_bytes()).hexdigest() != digest:
raise OSError(f"保存后读回校验失败:{path}")
if overwrite:
os.replace(temporary, path)
elif os.name == "nt":
# Windows rename is exclusive and also works on exFAT/FAT.
os.rename(temporary, path)
else:
# Publish only a complete file, without clobbering a racing writer.
os.link(temporary, path)
finally:
temporary.unlink(missing_ok=True)
return {
"path": str(targets[-1]),
"trace_path": str(trace_path),
"bytes": len(data),
"sha256": digest,
"verified": True,
}
async def _write(self, path: str, data: bytes, overwrite: bool) -> dict:
quoted = shlex.quote(path)
parent = str(PurePosixPath(path).parent)
remote_temp = f"{parent}/.artemis-{uuid.uuid4().hex}.tmp"
temp_quoted = shlex.quote(remote_temp)
self.root.mkdir(parents=True, exist_ok=True)
local_temp = self.root / f".artemis-{uuid.uuid4().hex}.tmp"
if not overwrite:
exists = await self._shell(
f"if test -e {quoted} || test -L {quoted}; then printf exists; fi"
)
if exists:
raise FileExistsError(f"设备文件已存在:{path};换名或获准后使用 overwrite=true")
await self._shell(f"mkdir -p {shlex.quote(parent)}")
digest = hashlib.sha256(data).hexdigest()
local_temp.write_bytes(data)
try:
await self._run("push", str(local_temp), remote_temp)
if await self._hash(remote_temp) != digest:
raise OSError("写入设备临时文件后校验失败,未替换目标文件")
# Toybox mv -n returns success when it skipped an existing target;
# the temporary file's continued existence makes that a failure.
flag = "-f" if overwrite else "-n"
await self._shell(
f"test ! -d {quoted} && test ! -L {quoted} && "
f"mv {flag} {temp_quoted} {quoted} && test ! -e {temp_quoted}"
)
if await self._hash(path) != digest:
raise OSError("设备目标文件读回校验失败")
return {"device_path": path, "bytes": len(data), "sha256": digest, "verified": True}
finally:
local_temp.unlink(missing_ok=True)
# Remove only this operation's own temporary file, even on failure.
try:
await self._shell(f"rm -f {temp_quoted}")
except (OSError, ValueError, TimeoutError):
pass
async def execute(self, args: "AndroidFileArgs") -> dict:
if args.action == "screenshot":
name = _filename(args.filename or f"android-{uuid.uuid4().hex[:12]}.png")
if Path(name).suffix.lower() != ".png":
raise ValueError("原生截图保存为 PNG,filename 必须以 .png 结尾")
data = await self._run("exec-out", "screencap", "-p", limit=MAX_FILE_BYTES)
with Image.open(BytesIO(data)) as image:
if image.format != "PNG":
raise ValueError("设备没有返回有效 PNG 截图")
image.load()
dimensions = {"width": image.width, "height": image.height}
return {
**self._save(data, name, args.destination, args.overwrite),
**dimensions,
"captured_at": datetime.now(UTC).isoformat(),
"device_serial": self.serial,
}
path = _device_path(args.device_path)
if args.action == "list":
return {
"device_path": path,
"entries": await self._shell(f"ls -la {shlex.quote(path)}"),
}
if args.action == "read_text":
data = await self._read(path, MAX_TEXT_BYTES)
return {"device_path": path, "content": data.decode("utf-8-sig"), "bytes": len(data)}
if args.action == "pull":
data = await self._read(path, MAX_FILE_BYTES)
return self._save(
data, args.filename or PurePosixPath(path).name, args.destination, args.overwrite
)
if args.action == "write_text":
if args.content is None:
raise ValueError("write_text 需要 content(UTF-8 文本,可为空字符串)")
data = args.content.encode("utf-8")
if len(data) > MAX_TEXT_BYTES:
raise ValueError("文本超过 64 KiB,请使用 push 传输现有文件")
else: # push: only a named artifact or a file explicitly selected on the desktop
root = self.root if args.destination == "trace" else desktop_directory()
source = _host_path(root, args.filename)
if not source.is_file() or source.stat().st_size > MAX_FILE_BYTES:
raise ValueError("本机源文件不存在或超过 64 MiB")
with source.open("rb") as handle:
data = handle.read(MAX_FILE_BYTES + 1)
if len(data) > MAX_FILE_BYTES:
raise ValueError("本机源文件超过 64 MiB")
return await self._write(path, data, args.overwrite)
class AndroidFileArgs(BaseModel):
action: Literal["screenshot", "list", "read_text", "write_text", "pull", "push"] = Field(
description="screenshot captures Android to host; list/read_text/write_text use Android paths; pull downloads, push uploads."
)
device_path: str = Field(
default="",
description="Absolute Android path, e.g. /sdcard/Download/result.txt. Not a PC path. Unused for screenshot.",
)
filename: str = Field(
default="",
description="Host file name only, no directories. Screenshot: .png (generated if empty); pull: defaults to device basename; push: required existing file.",
)
destination: Literal["trace", "desktop"] = Field(
default="trace",
description="Host location. Default: current task trace/artifacts/android. desktop: current user's actual desktop, only if requested; downloads also keep a trace copy. For push this selects the source location.",
)
content: str | None = Field(
default=None,
description="UTF-8 text for write_text only, max 64 KiB; empty string creates an empty file.",
)
overwrite: bool = Field(
default=False,
description="Only true when the user authorized replacing an existing file; default never overwrites.",
)
async def _android_file(ctx: Any, **kwargs: Any) -> str:
try:
args = AndroidFileArgs.model_validate(kwargs)
result = await AndroidFiles(ctx).execute(args)
return json.dumps({"ok": True, "action": args.action, **result}, ensure_ascii=False)
except TimeoutError:
return ToolFailure(f"安卓文件操作超时({ADB_TIMEOUT_SECONDS} 秒),ADB 子进程已停止")
except (OSError, ValueError) as exc:
return ToolFailure(f"安卓文件操作失败:{exc or type(exc).__name__}")
class AndroidFileTool(ArtemisTool):
def is_available(self, ctx: Any = None) -> bool:
return android_files_available(ctx)
async def execute(self, driver=None, ctx=None, **kwargs):
return await _android_file(ctx, **kwargs)
android_file = AndroidFileTool(
name="android_file",
description=(
"Save Android screenshots and read/write/transfer Android files on the task's selected device. "
"Use action=screenshot to save the CURRENT phone screen as native PNG, without UI shortcuts "
"or the computer-use plugin. Default host location is this task's trace/artifacts/android; "
"use destination=desktop only when the user requests the PC desktop. Returns verified paths, "
"size and SHA-256; never claim success without a successful result. Screenshot capture does "
"not prove the page is correct. Capture only after intended UI actions have executed and "
"the resulting screen has been observed, not alongside pending UI actions. "
"list/read_text/write_text operate on absolute ANDROID paths; "
"text is UTF-8, max 64 KiB. pull/push transfer a single file up to 64 MiB. No root or app-private "
"storage bypass; report Android permission errors. Existing files are protected by default."
),
args_schema=AndroidFileArgs,
category="system",
)
+9 -1
View File
@@ -14,6 +14,14 @@
> 搜索昨天包含“登录”的失败任务,根据截图与步骤说明失败原因,不重跑。
> 把当前安卓屏幕保存成 PNG 到本次任务的 trace 目录,并再保存一份到电脑桌面,返回实际文件路径。
Flash 和 Pro 都支持任务内的安卓文件工具 `android_file`:保存原生截图、列目录、读写 UTF-8 文本、在手机与任务目录/桌面之间传输文件。默认保存位置为 `traces/<trace_id>/artifacts/android/`,指定桌面时保留 trace 副本;Windows 会识别重定向的真实桌面。文本上限 64 KiB,单文件传输上限 64 MiB,每次 ADB 操作最多 60 秒,取消或超时会回收子进程。默认不覆盖已有文件,也不绕过 Android 文件访问权限。保存和传输会校验实际内容,返回文件路径、大小与 SHA-256;用例是否通过仍需结合画面和预期判断。
新保存的截图及下载文件会进入单任务“全部资料”和批次 ZIP 的 `任务文件/安卓/`。原有自动步骤截图仍位于 `traces/images/`,按步骤截图下载;二者用途不同,历史截图不能作为当前画面的新截图。
文件操作结果会进入任务执行记录,Flash 中也逐项记录操作步骤及校验结果,供终检和报告引用;不会只留下模型自行撰写的“已完成”说明。
## MCP 能力
| 工具 | 功能 |
@@ -38,7 +46,7 @@ Python 按 `uv.lock` 同步缺失/变化的依赖;前端依据源码、资源
scrcpy 固定版本的国内下载采用多个来源,失败或 SHA256 不符时自动尝试下一来源,全部失败会显示 HTTP 状态、超时或校验原因。`config/downloads.json` 的 `overrides.scrcpy_url` 指定首选 HTTPS 地址,`overrides.scrcpy_fallback_urls` 指定备用地址数组;`null` 使用默认备用源,`[]` 禁用备用源。国内模式不自动回退到 GitHub 直连。
MCP 同步仅检查 `$CODEX_HOME/config.toml`;未设置或为空时使用当前用户的 `~/.codex/config.toml`。日志会显示实际路径,以及新装、更新、已是最新、未检测到或自定义跳过的结果。检测到目录但缺少 ARTEMIS 条目时安装当前源码的标准配置;已有标准配置即使指向旧解压目录,也会更新解释器、项目目录、环境路径和工具清单。保留用户额外环境变量、禁用状态及其他服务;自定义启动包装器保持原样并提示。其他客户端的配置和规则文件不读取、不修改,其格式错误不会阻止启动。Codex 配置无法安全解析时停止启动,原文件不被覆盖;可使用 `-SkipMcpSync` 仅启动网页。首次修改保留备份。同步后需在 Codex 中重新加载 MCP 或重启 Codex,脚本无法替宿主重启已加载的 MCP 进程。
MCP 同步仅检查 `$CODEX_HOME/config.toml`;未设置或为空时使用当前用户的 `~/.codex/config.toml`。日志会显示实际路径,以及新装、更新、已是最新、未检测到或自定义跳过的结果。检测到目录但缺少 ARTEMIS 条目时安装当前源码的标准配置;已有标准配置即使指向旧解压目录,也会更新解释器、项目目录、环境路径和工具清单。保留用户额外环境变量、禁用状态、`tools` 下的工具审批策略及其他服务;不会重建权限表、添加 `eager` 或删除旧工具策略。自定义启动包装器保持原样并提示。其他客户端的配置和规则文件不读取、不修改,其格式错误不会阻止启动。Codex 原配置无法解析,或同步程序生成的更新未通过校验时,均停止启动且不覆盖原文件;日志会区分这两种原因,不把同步生成错误归因于用户配置。遇到后者请更新 ARTEMIS,不要删除有效的 `config.toml`;可使用 `-SkipMcpSync` 临时仅启动网页。首次修改保留备份。同步后需在 Codex 中重新加载 MCP 或重启 Codex,脚本无法替宿主重启已加载的 MCP 进程。
创建批次时必须提供唯一 `request_id`;请求超时后可以按编号查询或用相同内容重发,避免重复用例。创建只保存草稿,`start` 才执行。AI 无需先制造 Excel,也可直接提交 JSON;字段与示例见 [Skill 批量任务参考](../skills/artemis-automation/references/batches.md)。
+108
View File
@@ -0,0 +1,108 @@
# ARTEMIS 2.0.0:插件步骤、录像与开发规范
状态:2026-10-15 用户已确认设计,待实现。本文补充 [2.0.0 总体方案](2.0.0-plugins.md),不代表运行代码已具备这些能力。本轮只更新设计文档。
本次确认覆盖统一步骤、跨端录像、证据导出和插件开发规则。用户对审阅稿补充了三条约束:录像以高效留档为主;第三方插件和 Skill 必须按 ARTEMIS 规则适配;不自动恢复业务环境,恢复需求由任务提示词表达。
## 1. 统一步骤与结果
- 沿用一条任务时间线。步骤卡标明安卓、电脑、终端等来源,展示实际动作、结果、耗时和证据。观察、读取文件等辅助调用默认折叠,可展开。
- 以用户能理解的业务操作作为主步骤,其下保留实际工具调用、连点子动作和所有重试。分组只影响展示,不能隐藏失败或修改原始执行记录。
- 平台生成任务、步骤、调用和证据关联标识及时间;插件返回真实操作结果。一个业务步骤可以关联多次调用和多项附件。
- 区分“操作已执行”和“效果已验证”。工具成功不能直接变成用例通过;验收结合提示词和实际证据,保留不通过、未验证、阻塞等结果及人工复验优先级。
- 视觉步骤关联同一目标的操作前后截图,默认展示操作后图。缺图时明确说明,不用另一来源的画面补位。
- 每份视觉证据绑定插件及版本、资源实例、屏幕或窗口目标、采集时间、尺寸和坐标范围。窗口、屏幕、DPI 或目标切换后重新观察,不能仅按插件 ID 复用旧画面。
- 手机截图不能证明电脑操作成功,电脑截图不能进入手机观察状态。允许文字观察,但没有有效图像时不能猜测像素坐标。
## 2. 录像范围与生命周期
- 本次开发同时覆盖安卓和电脑录像;首批仍只有一个电脑控制插件,浏览器作为电脑应用操作,不增加独立浏览器插件。
- 通用协议预留其他视觉来源;终端、文件和 API 能力使用日志、退出码、请求结果、文件校验等原生证据,不强制提供录像。
- 录屏总开关继续默认关闭。用户开启后,自动覆盖任务实际使用且支持录制的来源,不要求选择执行目标或组合模式。
- 在来源首次被实际使用前启动录制,持续至任务暂停或结束。切换去操作另一来源时继续录,保留后台变化与等待过程;未使用来源不录。
- 暂停结束当前片段;恢复重新获取资源和观察,为后续使用建立新片段。任务结束或取消时收尾已录内容。
- 当前资源互斥是任务级,不随工具切换释放。“预留资源”和“开始录制”分别管理,不能因为插件可用就录完整个任务。
- 电脑默认录当前控制的目标屏幕,覆盖菜单、弹窗和窗口切换;提供仅录目标窗口的选项。多屏不默认全录,跨屏操作建立对应来源片段。
- 界面显示录制状态与当前屏幕/窗口范围,明确屏幕模式包含该屏幕其他可见内容。范围变化记录为事件,不悄悄扩大捕获范围。
## 3. 高效留档策略
目标是便于留档、体积可控、不会明显拖慢测试;录像仍需辨认操作和主要状态,细节文字可结合步骤截图查看。
- 优先评估 H.265/HEVC,优先利用可用的硬件编码能力;限制分辨率、帧率和目标码率,不默认保存原生高分辨率、高帧率视频。
- 编码能力、实际录制和网页播放兼容性必须验证;保留 H.264 回退,记录最终采用的编码及回退原因,不因强制 H.265 阻断任务。
- 实现时以长边不超过 1280、最高 15 fps、H.265 约 1 Mbps / H.264 约 1.5 Mbps 作为初始试验档位。这些是待测候选值,不是已验证或已生效的默认值;根据中文界面可读性、快速动作、CPU/GPU 占用和文件大小确定最终留档默认值。
- 优先录制时直接采用留档参数,避免任务结束或打包下载时再全量转码。已有合适片段尽量不重复编码。
- 默认只保留一套留档视频,不同时持久保存高清原片和压缩副本。临时转码文件按生命周期清理;用户证据不因清理被删除。
- 模型观察截图、步骤证据截图与录像参数独立,不能为了降低录像体积而降低模型定位所需画面质量。
- 编码、尺寸、帧率和码率进入统一配置层,提供合理默认;普通用户不必填写编码参数。网页、任务覆盖、MCP 和配置文档保持一致。
- 限帧或可变帧率不能破坏时间映射。实际时间戳、片段起止和关键帧定位必须用于回看,不能仅按帧数推算时间。
技术依据:[scrcpy 视频参数](https://github.com/Genymobile/scrcpy/blob/master/doc/video.md)提供尺寸、帧率、码率及编码器选项;[MDN 视频编码说明](https://developer.mozilla.org/en-US/docs/Web/Media/Guides/Formats/Video_codecs)用于核对播放兼容性。以项目实际安装版本和运行环境的能力探测及实测为准,不将文档列出的选项等同于所有设备均可用。
## 4. 回看与异常呈现
- 一个播放器区域切换安卓、电脑等来源,共享真实任务时间轴。首版不做多画面同时播放,不默认合并成一个大视频。
- 点击步骤或关键节点,自动切换对应来源并定位。跳转前正在播放则继续播放,暂停则保持暂停;按钮状态与实际播放器状态一致。
- 暂停、尚未录制、中断和无覆盖区间明确展示,不能抹掉缺口或用旧帧冒充连续画面。无视频步骤展示其截图或日志及缺失原因,不跳到无关片段。
- 普通录制故障告警后可以继续任务;要求以录像验收但缺少证据的项目标记未验证,不自动推断通过。
- 无法观察当前目标会影响视觉动作,应停止依赖该画面的操作,不能把它当成普通录屏故障继续盲点。
- 磁盘不足、编码失败等同类故障只产生一次主要告警,并持续显示录制状态;故障恢复有明确预算,不无界重试或阻塞批次。
## 5. 证据与导出
- 单任务和批次资料包含步骤记录、各来源截图与录像、插件日志、生成文件及统一索引。索引关联步骤、调用、来源、片段和视频时间,也说明缺失证据的原因。
- 截图和导出文件默认先留存到任务 trace;用户要求保存桌面等位置时额外复制,并记录真实写入结果及校验信息,不能仅凭工具文字声称保存成功。
- 批次按用例编号、轮次组织,关联任务 ID 以消除同名歧义;来源文件不相互覆盖。
- 网页、Word、Excel、单任务包和批次包的结论及证据引用一致。非视觉输出保留原生文件和可读摘要。
- 下载沿用现有打包进度。媒体压缩不挪到下载请求中执行,避免用户长时间等待;合适的已编码视频直接纳入资料包。
- 插件升级或卸载不影响历史证据读取。附件保留与删除遵循现有任务记录生命周期和共享文件引用规则。
## 6. 平台与插件的职责
- 平台负责配置、调度、版本固定、资源锁、权限范围检查、步骤和证据保存、时间映射、任务预算及报告。
- 插件负责自身能力的实际执行、观察、结果及附件输出,明确说明完成、失败、部分执行或无法确认,不自行判定整个测试用例通过,也不另建业务任务队列。
- 标准采用公共协议与能力附录:视觉、终端、API、文件各有证据要求,不把全部插件强制做成截图点击模式。
- 结果和附件协议明确类型、来源、时间、元数据与落库方式。接口版本、兼容范围和迁移规则必须与现有严格清单校验一起设计,不直接向旧清单塞入未知字段。
- 工具声明包含类型化参数、读写影响、资源范围、观察类型、截图/录像支持、超时、取消保证、重试与幂等语义。自动重复调用的观察工具必须明确只读,并接受校验。
- 未来终端插件可以明确提供命令能力,但应声明主机、工作目录与权限边界;普通业务插件不能用一个无限制 shell 接口代替全部规范。
## 7. 外部插件与 Skill 适配
- 既有开源插件和 Skill 必须按 ARTEMIS 规则适配并验证,再进入市场;不能直接复制即视为兼容。
- 可以保留适用的原工具实现,但工具名称与参数、资源身份、观察与坐标、权限、步骤和证据、错误和取消语义都要符合宿主契约;不兼容部分需要改造。
- Skill 描述实际存在的能力、使用方法、判断规则和限制。安装检查其所依赖的工具、平台及版本;缺少依赖应报清楚原因,不能靠提示词虚构能力。
- Skill 不能绕开授权、调度或证据规则,不能把工具返回成功直接写成业务通过。
- 保留原有来源和必要许可证信息;适配包按自己的版本和实测范围发布。
## 8. 失败处理与恢复边界
- 错误说明动作是否已执行、是否产生副作用、失败原因、受影响资源及已取得的部分证据。需要稳定错误码、可否重试及是否重新观察的语义,不能只有一条笼统失败消息。
- 明确未执行的请求可在预算内按策略重试;结果不明的保存、发送、删除等动作先核对,不盲目重复提交。模型重试不能变成业务用例重提。
- 沿用插件启动 30 秒、工具 60 秒、关闭 15 秒的配置默认,同时受任务总预算限制;现有参数可以配置,不新增无限等待。
- 远端动作无法确认停止时保护受影响资源,保留现有核对解除机制;不能因本地取消就假定远端已经停止。
- **不自动恢复业务环境,不建设默认原值快照、恢复计划或统一回滚模块。只有任务提示词提出恢复要求时,才作为普通任务步骤执行和验证。**
- 正常的连接关闭、录像收尾、进程回收和资源锁释放仍然必需。这些属于运行时清理,不是恢复用户业务配置。
## 9. 市场准入、开发资料与验收
- 市场仅发布按规则适配且完成相应验证的版本。覆盖多少平台和场景就声明多少,未覆盖项明确标为未验证或预览。
- 公共验收覆盖安装和兼容性、输入输出契约、错误/部分结果、超时取消、资源互斥、版本固定、证据导出和生命周期清理。
- 视觉插件额外验证窗口遮挡、DPI、多屏、窗口/目标切换、截图来源、坐标映射,以及声明支持的录像生命周期和回看。
- 录像验收覆盖长程体积、编码开销、中文可读性、快速动作、暂停恢复、丢失区间、多端时间映射、关键节点跳转、兼容回退和磁盘不足。
- 文件插件覆盖中文路径、覆盖保护和传输校验;终端/API 插件按其能力验证原生结果和副作用,不要求跑视觉专属项目。
- 提供最小可运行模板、公共规范、能力示例和自动合规检查命令;示例包含执行、网页展示和资料导出的完整路径。
- 延续 HTTPS、文件校验、平台兼容、安全解压与同版本不可覆盖规则。升级由用户发起,排队、暂停和运行任务固定版本;同一资源整条任务串行。
- 沿用用户启用时授权的范围,不新增逐动作确认;密钥不能进入普通查询、日志和报告。参数范围检查不宣称是操作系统沙箱。
## 10. 待实施顺序与检查点
- [ ] 定义公共结果/附件/时间契约及各能力声明,补齐错误与副作用语义和兼容迁移。
- [ ] 完成 Flash/Pro 的来源隔离与步骤分组,验证默认安卓任务调用插件、同插件跨窗口及多图证据。
- [ ] 实现电脑录像、统一录制生命周期和高效留档参数,实测后确定默认值及兼容回退。
- [ ] 实现多来源回看、缺口展示、关键节点与播放状态同步。
- [ ] 完成单任务/批次导出索引、原生附件和各报告一致性。
- [ ] 按 ARTEMIS 规则适配首发插件与 Skill,交付模板、能力附录及合规检查。
- [ ] 完成确定性检查、真实环境和长期运行验收,记录实测范围再发布。
已有静态检查发现两项需要复现验证的风险:Pro 对默认安卓任务调用插件时的图片隔离可能不完整;插件前图缓存按插件标识而非具体目标区分,跨窗口可能引用旧图。它们是待验证项,不应在复现前写成已确认故障或已修复。
+6 -4
View File
@@ -2,6 +2,8 @@
状态:核心插件链路、首发 Windows 插件适配已实现;Flash/Pro 纯电脑任务和基础浏览器表单实测通过,跨端与长期稳定性验收仍在进行,不是正式发行。
2026-10-15 已确认下一阶段的[插件步骤、录像与开发规范](2.0.0-plugin-evidence.md)。该文档是待实现的设计,包含电脑录像、统一回看、节省空间的留档策略及插件适配要求,不代表这些功能已经上线。
开发分支:`codex/v2.0.0-plugins`,从 1.5.2 主线建立。版本号为 2.0.0,发布前不创建正式版本标签。1.5.x 的 BUG 修复继续在主线进行,后续正常合并进开发分支。
插件市场仓库:<https://git.playones.com/chenmingxuan/Artemis-Plugins>
@@ -31,7 +33,7 @@
- 用户启用插件时授权其控制范围,范围内自动执行;超出范围等待用户指导。
- 执行层校验可调用工具及可识别的目标参数。第三方进程的系统权限必须如实说明,工具参数检查不能宣称等同于操作系统沙箱。
- AI 从已启用且配置就绪的插件中按需选择,按需加载工具、Skill 及其引用资源。
- 复用第三方 Skill 时适配工具名称和运行环境。依赖宿主专用 API 的 Skill 需要对应适配,不能仅复制文件就宣称可用。
- 复用第三方插件或 Skill 时,必须按 ARTEMIS 的工具、资源、权限、步骤、证据及错误契约适配并验证。可保留适用的原工具实现;依赖宿主专用 API 的 Skill 需要对应适配,不能仅复制文件就宣称可用。
## 任务执行和共享资源
@@ -48,10 +50,10 @@
## 环境恢复和报告
- 环境恢复由任务提示词或配置决定,不统一强制回滚。
- 具备读取能力时记录配置修改前后的值;要求恢复时将恢复纳入结束阶段并验证,失败单独展示并标记受影响资源。
- 不自动恢复业务环境,不建设默认的原值快照或回滚流程。只有用户任务提示词要求恢复时,才按任务执行并验证恢复;配置不会另行触发恢复。
- 按提示词执行的恢复失败时,在对应步骤和报告中如实记录。插件连接、录像进程及资源锁仍按正常生命周期清理。
- 手机、电脑和浏览器操作进入同一时间线,记录资源、插件版本、耗时、动作结果及证据来源。
- 首版电脑证据以步骤截图和工具结果为主。保留 Android 录屏,插件接口允许返回视频附件,电脑持续录屏后续扩展。
- 当前电脑证据以步骤截图和工具结果为主,已有 Android 录屏。已确认将电脑持续录屏、多来源同步回看和统一证据附件纳入本次 2.0.0 后续开发,按上述专项设计实施;目前尚未实现。
- 执行完成与测试通过继续分开。缺少验收证据时记录未验证或阻塞,保留已有明确结论,人工复验沿用现有优先级。
- 网页、Word、Excel、单任务资料包和批次资料包提供一致的跨端结果与证据。
- 对外 MCP、参数文档和随源码分发的自动化 Skill 同步新任务配置及能力查询。
+1 -1
View File
@@ -48,7 +48,7 @@ SDK:先 `await agent.init(execution_target="computer")`,然后 `agent.new_ta
暂停会先取消调用并关闭插件连接,再释放任务资源;恢复排队后重新观察。插件取消/超时的写操作可能已经生效,不能盲目重放。如果远程服务未确认动作停止,页面会显示资源保护及原因,用户核对后点击「已核对,解除保护」。这只解除 ARTEMIS 的保护,不会替用户终止远程操作。
工具返回成功表示调用完成,业务是否通过仍需实际证据。图片、工具文本、插件版本和动作结果进入任务记录;已有报告、截图下载和人工复验入口继续使用。Word 的「扩展插件操作」与 Excel 的「插件操作」保留来源及实际结果,资料包截图名称含插件 ID。同一截图被多个任务引用时,删除最后一个任务后才清理。插件画面与手机截图属于不同来源,不能相互替代。电脑持续录屏不属于首版能力。
工具返回成功表示调用完成,业务是否通过仍需实际证据。图片、工具文本、插件版本和动作结果进入任务记录;已有报告、截图下载和人工复验入口继续使用。Word 的「扩展插件操作」与 Excel 的「插件操作」保留来源及实际结果,资料包截图名称含插件 ID。同一截图被多个任务引用时,删除最后一个任务后才清理。插件画面与手机截图属于不同来源,不能相互替代。电脑持续录屏目前尚未实现,已纳入[步骤、录像与插件开发规范设计](plans/2.0.0-plugin-evidence.md);该设计不是当前版本的能力声明。
## 插件清单
+24 -11
View File
@@ -18,6 +18,10 @@ from urllib.parse import urlsplit
ROOT = Path(__file__).resolve().parents[1]
class McpSyncValidationError(ValueError):
"""The synchronizer could not safely update an already parsed configuration."""
def launch_text(english: str, chinese: str) -> str:
return chinese if os.environ.get("ARTEMIS_LAUNCH_LANG") == "zh-CN" else english
@@ -170,11 +174,10 @@ def _updated_entry(config: dict, template: dict, root: Path, names: tuple[str, .
updated["env"] = {**environment, **template["env"]}
if "enabled_tools" in template or "enabled_tools" in config:
updated["enabled_tools"] = list(names)
if "tools" in template or "tools" in config:
current_tools = config.get("tools", {})
if not isinstance(current_tools, dict):
raise ValueError("The existing Artemis MCP tools setting is not an object")
updated["tools"] = {name: {"eager": True, **current_tools.get(name, {})} for name in names}
# Per-tool approval/enablement policies belong to the user. Leave `tools`
# untouched, including entries outside the current exported tool list.
# Rebuilding it both changes policy and sends nested tables to the scalar
# serializer, which produces JSON syntax rather than valid TOML.
for key in ("type", "required", "startup_timeout_sec", "connectionTimeoutMs"):
if key in template and key not in updated:
updated[key] = template[key]
@@ -300,12 +303,17 @@ def refresh_codex(
assignments[env_header] = env_updates
if not top and not env_updates:
return False
result = _replace_toml_tables(raw, assignments)
actual = tomllib.loads(result)["mcp_servers"]["artemis"]
if any(actual.get(key) != value for key, value in top.items()) or any(
actual.get("env", {}).get(key) != value for key, value in env_updates.items()
):
raise ValueError("Could not safely refresh the Artemis MCP table")
try:
result = _replace_toml_tables(raw, assignments)
actual = tomllib.loads(result)["mcp_servers"]["artemis"]
if any(actual.get(key) != value for key, value in top.items()) or any(
actual.get("env", {}).get(key) != value for key, value in env_updates.items()
):
raise ValueError("Updated MCP settings did not match the requested values")
except (ValueError, KeyError, TypeError) as exc:
raise McpSyncValidationError(
"Could not safely generate the MCP configuration update"
) from exc
path.parent.mkdir(parents=True, exist_ok=True)
_atomic_write(path, result)
return result != raw
@@ -339,6 +347,11 @@ def refresh_mcp(root: Path) -> int:
"Cannot read/write the configuration; check file permissions.",
"无法读写配置,请检查文件权限。",
)
elif isinstance(exc, McpSyncValidationError):
reason = launch_text(
"The synchronizer generated an invalid update from a readable configuration. This is an ARTEMIS sync error; do not delete your Codex configuration.",
"原配置可正常解析,但同步程序生成的更新未通过校验。这是 ARTEMIS 同步错误,请勿删除 Codex 配置。",
)
else:
reason = launch_text(
"Invalid TOML or Artemis configuration structure; check the file in Codex.",
+7
View File
@@ -35,6 +35,13 @@ Windows 启动脚本只提示源码更新。升级使用 `update.bat`(国内
- 执行状态与业务判定分开:`completed` 只说明执行结束,不等于测试通过。Flash 和 Pro 默认根据任务提示词、用户指导、执行结论和实际证据进行只读终检,不需要另填预期结果。读取 `test_result.status`(`passed` 通过、`failed` 不通过、`inconclusive` 未验证、`blocked` 阻塞)、`reason` 与 `evidence`,结合验证摘要和 `mobile_inspect_trace` 的步骤/截图交付结果。执行失败不能直接认定产品缺陷;已确认的断言失败不得被后续成功覆盖。终检关闭、出错或证据不足时不得宣称通过。
- 用 `mobile_export_report(session_id=...)` 获取单任务报告;`formats=["screenshots"]` 获取排序截图,`["video"]` 获取保存的录屏分段,`["all"]` 获取 Word、Excel 和所有可用资料的 ZIP。批次传 `batch_id` 获取汇总、各项任务报告、步骤截图及已有录屏的 ZIP。未指定格式时使用保存的 Word/Excel 偏好。执行中的导出只是当前快照。
- 工具返回 MCP 主机上的绝对文件路径和下载 URL。向用户交付真实生成的文件链接,并说明完成、失败、未验证、跳过数量及重要限制。不要只说“报告已生成”而不交付文件。
### 安卓截图和文件
- Flash/Pro 任务内部均提供 `android_file`,由任务已分配的 Android 设备执行。外层 MCP 使用现有 `mobile_run_task` 提交目标,不直接绕过任务队列传文件。
- 保存当前手机画面使用 `action="screenshot"`;原生 PNG 默认位于 `traces/<trace_id>/artifacts/android/`。明确要求保存电脑桌面时用 `destination="desktop"`,同时保留 trace 副本;无需电脑插件或截图快捷键。自动步骤截图原本保存在 `traces/images/`,不能把旧截图当作新的截屏结果。
- `list`、`read_text`、`write_text` 的 `device_path` 是 Android 绝对路径;文本使用 UTF-8,单次不超过 64 KiB。`pull` 下载手机文件,`push` 将 trace 中的文件或用户指定的桌面文件上传手机,单文件不超过 64 MiB。`filename` 仅是文件名,不能传 Windows 路径;目录由 `destination` 选择。不能绕过 Android 的应用私有目录权限。
- 保存、上传和下载会校验内容,已有文件默认不覆盖。只有工具成功并返回真实路径/校验结果后才可宣称保存成功;截图内容是否符合测试预期仍需核对。单任务“全部资料”及批次 ZIP 包含这些文件,位于 `任务文件/安卓/`。
- 用 `mobile_search_history` 按名称、目标、编号或标签搜索全部历史;日期先按用户时区转换为 Unix 秒,`start` 含边界、`end` 不含边界。按返回的 `total/page/size` 翻页,不能把第一页当作全部历史。
- `mobile_inspect_trace` 是只读步骤证据查看;其中历史上下文的 replay 术语不表示重新执行设备动作。
@@ -988,6 +988,10 @@ def test_task_material_zip_contains_ordered_evidence_and_reports(
):
job = isolated.jobs(service.create_batch(payload())["id"])[0]
isolated.save_job({**job, "case": {**job["case"], "case_id": "1.1"}})
artifact_dir = reports.TRACES_PATH / job["session_id"] / "artifacts" / "android"
artifact_dir.mkdir(parents=True)
(artifact_dir / "安卓截图.png").write_bytes(b"native png")
(artifact_dir / ".artemis-partial.tmp").write_bytes(b"incomplete")
evidence = tmp_path / "evidence"
evidence.mkdir()
first = evidence / "first.png"
@@ -1015,9 +1019,37 @@ def test_task_material_zip_contains_ordered_evidence_and_reports(
assert archive.read("1.1/任务录屏/0001_recording.mp4") == b"recording.mp4"
manifest = json.loads(archive.read("1.1/文件清单.json"))
assert manifest["session_id"] == job["session_id"]
assert manifest["artifacts"] == ["安卓截图.png"]
assert archive.read("1.1/任务文件/安卓/安卓截图.png") == b"native png"
assert not any("partial" in name for name in files)
assert "1.1-all.zip" in response.headers["content-disposition"]
def test_batch_zip_contains_each_tasks_android_artifacts(isolated):
batch = service.create_batch(payload())
for index, job in enumerate(isolated.jobs(batch["id"])):
root = reports.TRACES_PATH / job["session_id"] / "artifacts" / "android"
root.mkdir(parents=True)
(root / "结果.txt").write_text(f"任务{index}", encoding="utf-8")
with zipfile.ZipFile(reports.export_batch(batch["id"])) as archive:
names = [name for name in archive.namelist() if name.endswith("任务文件/安卓/结果.txt")]
assert len(names) == 2
assert {archive.read(name).decode() for name in names} == {"任务0", "任务1"}
def test_android_artifact_export_does_not_follow_other_session_links(isolated):
root = reports.TRACES_PATH / "session" / "artifacts"
root.mkdir(parents=True)
other = reports.TRACES_PATH / "other-session"
other.mkdir()
(other / "private.txt").write_text("other task")
try:
(root / "android").symlink_to(other, target_is_directory=True)
except OSError:
pytest.skip("symlinks unavailable")
assert reports.session_artifacts({"id": "session"}) == []
def test_video_export_accepts_finalized_worker_trace_and_rejects_external_path(
isolated, tmp_path, monkeypatch
):
+1 -1
View File
@@ -151,7 +151,7 @@ def test_full_set_enum_slots_render_verbatim():
" `press_key`, `manage_app`, or `wait_for_delay`)" in out
)
assert "Helper/Subagent tools (`ask_explorer`, `ask_diagnoser`, `video_analyzer`)" in out
assert "ADB/task tools (`run_adb_command`, `manage_task`)" in out
assert "ADB/task tools (`run_adb_command`, `manage_task`, `android_file`)" in out
assert (
"(`read_note`, `list_notes`, `search_history`, `replay_steps`, `get_step_screenshot`)"
in out
@@ -197,6 +197,15 @@ async def test_perception_node_async_offloading(mock_artemis_ctx, tmp_path):
engine = DataEngine(mock_artemis_ctx)
engine.start_session("Test Perception Node Offloading")
mock_artemis_ctx.data_engine = engine
image_saved = asyncio.Event()
loop = asyncio.get_running_loop()
save_image = engine.get_or_create_image
def save_image_and_signal(*args, **kwargs):
try:
return save_image(*args, **kwargs)
finally:
loop.call_soon_threadsafe(image_saved.set)
# Create an injected_instruction.json file
instruction_file = Path(engine.base_dir) / "injected_instruction.json"
@@ -214,6 +223,7 @@ async def test_perception_node_async_offloading(mock_artemis_ctx, tmp_path):
mock_screen_data.height = 2400
with (
patch.object(engine, "get_or_create_image", side_effect=save_image_and_signal),
patch("artemis.graph.perception.UnifiedMobileController") as mock_controller_cls,
patch("artemis.graph.perception.perform_ocr", new_callable=AsyncMock) as mock_ocr,
patch("artemis.graph.perception._detect_status_bar_height", return_value=0),
@@ -240,10 +250,9 @@ async def test_perception_node_async_offloading(mock_artemis_ctx, tmp_path):
# Verify injected_instruction.json was unlinked
assert not instruction_file.exists()
# Await background tasks triggered by perception_node
await asyncio.sleep(0.1)
if engine._pending_tasks:
await asyncio.gather(*engine._pending_tasks)
# perception_node creates its own task, outside engine._pending_tasks.
# Wait for actual persistence instead of assuming the disk finishes in 100 ms.
await asyncio.wait_for(image_saved.wait(), timeout=5)
# Verify image record and file were created in background by perception_node
img_record = engine.storage.get_image(expected_hash)
+25 -8
View File
@@ -7,6 +7,7 @@ import logging
import os
from pathlib import Path
import threading
from types import SimpleNamespace
import pytest
@@ -35,8 +36,12 @@ def test_running_state_recovers_from_temporary_windows_file_access(tmp_path, mon
raise _windows_access_error(winerror)
return replace(source, target)
monkeypatch.setattr(task_control.os, "replace", busy_replace)
monkeypatch.setattr(task_control.time, "sleep", delays.append)
monkeypatch.setattr(
task_control, "os", SimpleNamespace(**{**vars(task_control.os), "replace": busy_replace})
)
monkeypatch.setattr(
task_control, "time", SimpleNamespace(**{**vars(task_control.time), "sleep": delays.append})
)
task_control.TaskControl("task-123", object())._state("running")
assert task_control.read_state("task-123")["status"] == "running"
@@ -55,8 +60,12 @@ def test_persistent_windows_access_error_is_bounded_and_preserves_old_state(tmp_
attempts.append(source)
raise error
monkeypatch.setattr(task_control.os, "replace", denied_replace)
monkeypatch.setattr(task_control.time, "sleep", delays.append)
monkeypatch.setattr(
task_control, "os", SimpleNamespace(**{**vars(task_control.os), "replace": denied_replace})
)
monkeypatch.setattr(
task_control, "time", SimpleNamespace(**{**vars(task_control.time), "sleep": delays.append})
)
with pytest.raises(PermissionError) as raised:
task_control._write_atomic(path, {"status": "running"})
@@ -82,8 +91,12 @@ def test_unrelated_file_errors_fail_without_retry_and_clean_temporary_file(
attempts.append(source)
raise error
monkeypatch.setattr(task_control.os, "replace", failed_replace)
monkeypatch.setattr(task_control.time, "sleep", delays.append)
monkeypatch.setattr(
task_control, "os", SimpleNamespace(**{**vars(task_control.os), "replace": failed_replace})
)
monkeypatch.setattr(
task_control, "time", SimpleNamespace(**{**vars(task_control.time), "sleep": delays.append})
)
with pytest.raises(OSError) as raised:
task_control._write_atomic(path, {"status": "running"})
@@ -127,7 +140,9 @@ def test_busy_temporary_cleanup_preserves_original_write_error(tmp_path, monkeyp
raise _windows_access_error(32)
return unlink(target, **kwargs)
monkeypatch.setattr(task_control.os, "replace", failed_replace)
monkeypatch.setattr(
task_control, "os", SimpleNamespace(**{**vars(task_control.os), "replace": failed_replace})
)
monkeypatch.setattr(Path, "unlink", busy_unlink)
monkeypatch.setattr(task_control.time, "sleep", lambda _: None)
try:
@@ -163,7 +178,9 @@ def test_repeated_control_requests_recover_from_file_sharing(
raise _windows_access_error(32)
return replace(source, target)
monkeypatch.setattr(task_control.os, "replace", busy_replace)
monkeypatch.setattr(
task_control, "os", SimpleNamespace(**{**vars(task_control.os), "replace": busy_replace})
)
monkeypatch.setattr(task_control.time, "sleep", lambda _: None)
send_request("task-123")
+307
View File
@@ -0,0 +1,307 @@
"""Android artifact integrity, path separation and Flash/Pro tool parity."""
import asyncio
import hashlib
from io import BytesIO
import json
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock
from PIL import Image
import pytest
from artemis.agents.flash.runner import FlashRunner
from artemis.agents.validator.tool_declarations import ToolExecutionResult
from artemis.core.tool_failure import is_tool_failure
from artemis.mcp.action_executor import McpActionExecutor
from artemis.tools import android_files as files
@pytest.fixture
def ctx(tmp_path):
return SimpleNamespace(
execution_target="mixed",
device=SimpleNamespace(device_id="task-device"),
data_engine=SimpleNamespace(base_dir=tmp_path / "trace", current_session_id="session"),
execution_setup=None,
actuator=None,
plugins=None,
)
@pytest.fixture
def png():
stream = BytesIO()
Image.new("RGB", (144, 312), "red").save(stream, "PNG")
return stream.getvalue()
@pytest.mark.asyncio
async def test_screenshot_saved_native_and_desktop_copy_verified(ctx, png, tmp_path, monkeypatch):
desktop = tmp_path / "重定向桌面"
desktop.mkdir()
monkeypatch.setattr(files, "desktop_directory", lambda: desktop)
run = AsyncMock(return_value=png)
monkeypatch.setattr(files.AndroidFiles, "_run", run)
result = json.loads(
await files.android_file.execute(
ctx=ctx, action="screenshot", filename="手机截图.png", destination="desktop"
)
)
assert result["ok"] and result["verified"]
assert result["width"] == 144 and result["height"] == 312
assert result["device_serial"] == "task-device"
assert Path(result["path"]) == desktop / "手机截图.png"
assert Path(result["path"]).read_bytes() == Path(result["trace_path"]).read_bytes() == png
assert result["sha256"] == hashlib.sha256(png).hexdigest()
run.assert_awaited_once_with("exec-out", "screencap", "-p", limit=files.MAX_FILE_BYTES)
@pytest.mark.asyncio
async def test_existing_artifact_is_preserved_unless_explicit_overwrite(ctx, png, monkeypatch):
monkeypatch.setattr(files.AndroidFiles, "_run", AsyncMock(return_value=png))
first = json.loads(
await files.android_file.execute(ctx=ctx, action="screenshot", filename="a.png")
)
path = Path(first["trace_path"])
path.write_bytes(b"original")
error = await files.android_file.execute(ctx=ctx, action="screenshot", filename="a.png")
assert is_tool_failure(error) and path.read_bytes() == b"original"
result = await files.android_file.execute(
ctx=ctx, action="screenshot", filename="a.png", overwrite=True
)
assert json.loads(result)["ok"] and path.read_bytes() == png
@pytest.mark.asyncio
@pytest.mark.parametrize(
"name", ["../escape.png", "C:\\out.png", "x:y.png", "CON.png", "x.png ", "a\n.png"]
)
async def test_invalid_host_paths_fail_before_adb(ctx, name, monkeypatch):
run = AsyncMock()
monkeypatch.setattr(files.AndroidFiles, "_run", run)
result = await files.android_file.execute(ctx=ctx, action="screenshot", filename=name)
assert is_tool_failure(result)
run.assert_not_awaited()
@pytest.mark.asyncio
@pytest.mark.parametrize(
"device_path", ["C:\\Desktop\\x", "relative", "/sdcard/../x", "/", "/sdcard/a\n"]
)
async def test_android_path_validation(ctx, device_path, monkeypatch):
run = AsyncMock()
monkeypatch.setattr(files.AndroidFiles, "_run", run)
assert is_tool_failure(
await files.android_file.execute(ctx=ctx, action="read_text", device_path=device_path)
)
run.assert_not_awaited()
@pytest.mark.asyncio
async def test_invalid_image_does_not_create_artifact(ctx, monkeypatch):
monkeypatch.setattr(files.AndroidFiles, "_run", AsyncMock(return_value=b"screencap failed"))
assert is_tool_failure(await files.android_file.execute(ctx=ctx, action="screenshot"))
assert not (ctx.data_engine.base_dir / "artifacts").exists()
@pytest.mark.asyncio
async def test_read_and_pull_verify_chinese_and_binary_data(ctx, monkeypatch):
service = files.AndroidFiles(ctx)
data = "中文内容\n第二行".encode()
digest = hashlib.sha256(data).hexdigest()
shell = AsyncMock(side_effect=[str(len(data)), digest, digest])
monkeypatch.setattr(service, "_shell", shell)
monkeypatch.setattr(service, "_run", AsyncMock(return_value=data))
result = await service.execute(
files.AndroidFileArgs(action="read_text", device_path="/sdcard/a '中'.txt")
)
assert result["content"] == data.decode()
assert "'\"'\"'" in shell.call_args_list[0].args[0] # POSIX shell quoting
binary = b"\x00\xff\r\n\x1a"
digest = hashlib.sha256(binary).hexdigest()
shell.side_effect = [str(len(binary)), digest, digest]
service._run.return_value = binary
result = await service.execute(
files.AndroidFileArgs(action="pull", device_path="/sdcard/file.bin")
)
assert Path(result["path"]).read_bytes() == binary
@pytest.mark.asyncio
async def test_changed_or_oversize_file_is_not_saved(ctx, monkeypatch):
service = files.AndroidFiles(ctx)
monkeypatch.setattr(service, "_shell", AsyncMock(side_effect=["3", "0" * 64]))
run = AsyncMock(return_value=b"abc")
monkeypatch.setattr(service, "_run", run)
with pytest.raises(OSError, match="传输"):
await service.execute(files.AndroidFileArgs(action="pull", device_path="/sdcard/a"))
assert not service.root.exists()
service._shell.side_effect = [str(files.MAX_TEXT_BYTES + 1)]
run.reset_mock()
with pytest.raises(ValueError, match="上限"):
await service.execute(files.AndroidFileArgs(action="read_text", device_path="/sdcard/a"))
run.assert_not_awaited()
@pytest.mark.asyncio
async def test_write_text_pushes_utf8_and_verifies_before_rename(ctx, monkeypatch):
service = files.AndroidFiles(ctx)
data = "第一行\n第二行".encode()
calls = []
async def run(*args, **kwargs):
if args[0] == "push":
assert Path(args[1]).read_bytes() == data
calls.append("push")
return b""
async def shell(command):
calls.append(command)
return hashlib.sha256(data).hexdigest() if command.startswith("sha256sum") else ""
monkeypatch.setattr(service, "_run", run)
monkeypatch.setattr(service, "_shell", shell)
result = await service.execute(
files.AndroidFileArgs(
action="write_text", device_path="/sdcard/Download/文本.txt", content=data.decode()
)
)
assert result["verified"] and result["bytes"] == len(data)
assert any("mv -n" in call and "test ! -e" in call for call in calls)
assert calls.index("push") < next(i for i, c in enumerate(calls) if "mv -n" in c)
assert not list(service.root.iterdir()) # no leaked local staging files
@pytest.mark.asyncio
async def test_permission_failure_propagates_and_does_not_claim_success(ctx, monkeypatch):
monkeypatch.setattr(
files.AndroidFiles, "_run", AsyncMock(side_effect=OSError("Permission denied"))
)
result = await files.android_file.execute(
ctx=ctx, action="read_text", device_path="/data/private.txt"
)
assert is_tool_failure(result) and "Permission denied" in result
@pytest.mark.asyncio
async def test_existing_android_file_is_never_pushed_over(ctx, monkeypatch):
service = files.AndroidFiles(ctx)
monkeypatch.setattr(service, "_shell", AsyncMock(return_value="exists"))
push = AsyncMock()
monkeypatch.setattr(service, "_run", push)
with pytest.raises(FileExistsError, match="设备文件已存在"):
await service.execute(
files.AndroidFileArgs(action="write_text", device_path="/sdcard/a", content="new")
)
push.assert_not_awaited()
@pytest.mark.asyncio
@pytest.mark.parametrize("failure", ["cancel", "timeout", "limit"])
async def test_adb_subprocess_is_reaped_on_interrupt(ctx, monkeypatch, failure):
service = files.AndroidFiles(ctx)
process = Mock(returncode=None)
process.stdout.read = AsyncMock()
process.stderr.read = AsyncMock(return_value=b"")
if failure == "limit":
process.stdout.read.return_value = b"too large"
expected = ValueError
else:
async def wait_forever(*args):
await asyncio.Event().wait()
process.stdout.read.side_effect = wait_forever
expected = asyncio.CancelledError if failure == "cancel" else TimeoutError
process.wait = AsyncMock(return_value=0)
def kill():
process.returncode = -1
process.kill.side_effect = kill
spawn = AsyncMock(return_value=process)
monkeypatch.setattr(asyncio, "create_subprocess_exec", spawn)
monkeypatch.setattr(files, "ADB_TIMEOUT_SECONDS", 0.02)
task = asyncio.create_task(service._run("exec-out", "cat /sdcard/a", limit=2))
await asyncio.sleep(0)
if failure == "cancel":
task.cancel()
with pytest.raises(expected):
await task
process.kill.assert_called_once()
process.wait.assert_awaited()
argv = spawn.call_args.args
assert argv[argv.index("-s") + 1] == "task-device"
assert "-H" in argv and "-P" in argv
@pytest.mark.asyncio
async def test_flash_and_pro_share_schema_handler_and_failure_status(ctx, png, monkeypatch):
monkeypatch.setattr(files.AndroidFiles, "_run", AsyncMock(return_value=png))
monkeypatch.setattr("artemis.tools.base.get_driver", lambda ctx: None)
monkeypatch.setattr(
"artemis.mcp.action_executor.TraceSpan",
lambda **kw: Mock(__enter__=lambda self: self, __exit__=lambda *a: False),
)
runner = object.__new__(FlashRunner)
runner.ctx = ctx
assert "android_file" in {t.name for t in runner._get_tools()}
pro = files.android_file.to_langchain_tool(ctx)
assert pro.args_schema is files.AndroidFileArgs
result = await pro.ainvoke({"action": "screenshot", "filename": "pro.png"})
assert json.loads(result)["verified"]
executor = McpActionExecutor(ctx, actuator=Mock())
result = await executor.execute(
"android_file", {"action": "screenshot", "filename": "flash.png"}, "call", None
)
assert result.status == "success" and json.loads(result.text_summary)["verified"]
result = await executor.execute(
"android_file", {"action": "screenshot", "filename": "flash.png"}, "call2", None
)
assert result.status == "error"
ctx.execution_target = "computer"
assert "android_file" not in {t.name for t in runner._get_tools()}
assert is_tool_failure(await files.android_file.execute(ctx=ctx, action="screenshot"))
@pytest.mark.asyncio
@pytest.mark.parametrize("status", ["success", "error"])
async def test_flash_records_file_receipts_as_steps_for_final_verification(ctx, status):
runner = object.__new__(FlashRunner)
runner.ctx = ctx
runner._llm = None
runner.summarizer = None
result = ToolExecutionResult(
tool_call_id="file-call",
tool_name="android_file",
status=status,
text_summary='{"verified": true, "sha256": "receipt"}'
if status == "success"
else "Permission denied",
)
runner.executor = SimpleNamespace(
action_tool_names=frozenset({"click"}), execute=AsyncMock(return_value=result)
)
runner._record_action_step = Mock(return_value="recorded-file-step")
turn = SimpleNamespace(step_keys=[], actions=[])
messages = []
await runner._execute_and_record_action(
name="android_file",
args={"action": "screenshot"},
tc_id="file-call",
state=SimpleNamespace(),
messages=messages,
raw_text="保存当前安卓屏幕",
step_token_usage={},
pre_screenshot_bytes=b"previous-observation",
xml_list="UI",
action_sequence=0,
turn=turn,
)
runner._record_action_step.assert_called_once()
assert runner._record_action_step.call_args.args[2] is result
assert turn.step_keys == ["recorded-file-step"]
assert turn.actions == [("android_file", status, result.text_summary)]
assert messages[-1].content == result.text_summary
+72
View File
@@ -136,6 +136,78 @@ command = "leave-alone"
assert path.with_name(path.name + ".artemis-update.bak").read_text(encoding="utf-8") == original
@pytest.mark.parametrize(
"policies",
[
'# Keep my tool policies\ntools = { mobile_run_task = { enabled = false, approval_policy = "on-request" }, retired_tool = { enabled = false } }\n',
'# Keep my tool policies\ntools.mobile_run_task.enabled = false\ntools.mobile_run_task.approval_policy = "on-request"\ntools.retired_tool.enabled = false\n',
'# Keep my tool policies\n[mcp_servers.artemis.tools.mobile_run_task]\nenabled = false\napproval_policy = "on-request"\n[mcp_servers.artemis.tools.retired_tool]\nenabled = false\n',
],
ids=["inline-table", "dotted-keys", "tool-subtables"],
)
def test_codex_sync_preserves_tool_policies_verbatim(tmp_path, monkeypatch, policies):
from artemis.interfaces.cli.commands import mcp as mcp_command
codex_home = tmp_path / "中文用户" / ".codex"
codex_home.mkdir(parents=True)
path = codex_home / "config.toml"
original = (
'[mcp_servers.artemis]\ncommand = "old-python"\n'
'args = ["-m", "mcp_server"]\n'
'enabled_tools = ["mobile_run_task"]\n'
'disabled_tools = ["mobile_config"]\n'
'env = { PYTHONPATH = "C:/old", OPENAI_API_KEY = "test-secret" }\n'
+ policies
+ '[mcp_servers.other]\ncommand = "leave-alone"\n'
)
path.write_text(original, encoding="utf-8")
before = tomllib.loads(original)
monkeypatch.setenv("CODEX_HOME", str(codex_home))
root = tmp_path / "新版项目"
assert update.refresh_mcp(root) == 0
refreshed = path.read_text(encoding="utf-8")
after = tomllib.loads(refreshed)
entry = after["mcp_servers"]["artemis"]
assert entry["command"] == update.sys.executable
assert entry["cwd"] == str(root)
assert entry["env"] == {
"PYTHONPATH": str(root),
"OPENAI_API_KEY": "test-secret",
"PYTHONUNBUFFERED": "1",
"ARTEMIS_DESKTOP_NOTIFY": "true",
}
assert entry["enabled_tools"] == list(mcp_command.ARTEMIS_MCP_TOOLS)
assert entry["disabled_tools"] == ["mobile_config"]
assert entry["tools"] == before["mcp_servers"]["artemis"]["tools"]
assert policies.split("\n", 1)[1] in refreshed
assert refreshed.count("# Keep my tool policies") == 1
assert after["mcp_servers"]["other"] == before["mcp_servers"]["other"]
assert path.with_name(path.name + ".artemis-update.bak").read_text(encoding="utf-8") == original
assert update.refresh_mcp(root) == 0
assert path.read_text(encoding="utf-8") == refreshed
def test_codex_sync_reports_generated_toml_error_without_blame_or_overwrite(
tmp_path, monkeypatch, capsys
):
codex_home = tmp_path / ".codex"
codex_home.mkdir()
path = codex_home / "config.toml"
original = '[mcp_servers.artemis]\ncommand = "old"\nargs = ["-m", "mcp_server"]\n'
path.write_text(original, encoding="utf-8")
monkeypatch.setenv("CODEX_HOME", str(codex_home))
monkeypatch.setenv("ARTEMIS_LAUNCH_LANG", "zh-CN")
monkeypatch.setattr(update, "_replace_toml_tables", lambda *_: 'tools = {"secret": true}\n')
assert update.refresh_mcp(tmp_path / "project") == 1
assert path.read_text(encoding="utf-8") == original
assert not path.with_name(path.name + ".artemis-update.bak").exists()
output = capsys.readouterr().out
assert "原配置可正常解析" in output
assert "ARTEMIS 同步错误" in output
assert "配置结构无效" not in output
assert "secret" not in output
def test_mcp_sync_updates_only_codex_and_preserves_other_clients(tmp_path, monkeypatch):
from artemis.interfaces.cli.commands import mcp as mcp_command