chenmingxuan
e1ea099592
Install and update artemis-use with task diagnostics and plugin guidance. Preserve cross-device file receipts, review canonical reports, guard stage transitions and reduce small history summaries. Improve exploration lifecycle, request recovery and browser live connections.
342 lines
14 KiB
Python
342 lines
14 KiB
Python
"""Helper-tool routing in McpActionExecutor (Flash profile).
|
|
|
|
The Flash runner binds ``ask_explorer``, ``video_analyzer`` and the history
|
|
tools (``search_history`` / ``replay_steps`` / ``get_step_screenshot``) next to its device actions; the executor must
|
|
route them to the same implementations the Pro operator's LangChain tools
|
|
delegate to, as agent-side text tools (no screenshot capture, no device
|
|
action), and report their status explicitly instead of sniffing the text for
|
|
the word "Error".
|
|
"""
|
|
|
|
from unittest.mock import AsyncMock, Mock, patch
|
|
|
|
import pytest
|
|
|
|
from artemis.mcp.action_executor import AGENT_TOOL_NAMES, McpActionExecutor
|
|
from artemis.tools.explorer_tool import ExplorerCandidate, ExplorerOutcome
|
|
|
|
|
|
def _make_executor(**kwargs):
|
|
ctx = Mock()
|
|
ctx.data_engine = None
|
|
ctx.device.device_width = 1080
|
|
ctx.device.device_height = 2400
|
|
actuator = Mock()
|
|
actuator.controller = Mock()
|
|
actuator.extensions.return_value = []
|
|
return McpActionExecutor(ctx, actuator=actuator, **kwargs)
|
|
|
|
|
|
def _state():
|
|
state = Mock()
|
|
state.latest_screenshot = "/tmp/shot.jpg"
|
|
state.indexed_points = []
|
|
state.indexed_elements = []
|
|
state.operator_raw_data = None
|
|
return state
|
|
|
|
|
|
def test_helper_tools_are_not_device_actions():
|
|
executor = _make_executor()
|
|
assert {
|
|
"ask_explorer",
|
|
"video_analyzer",
|
|
"search_history",
|
|
"replay_steps",
|
|
"get_step_screenshot",
|
|
} <= AGENT_TOOL_NAMES
|
|
assert not (AGENT_TOOL_NAMES & executor.action_tool_names)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_flash_saves_and_reads_text_with_verified_receipt_without_device_actions(tmp_path):
|
|
import hashlib
|
|
import json
|
|
|
|
executor = _make_executor(agent_name="flash")
|
|
executor.ctx.data_engine = Mock(base_dir=tmp_path, current_step_id=None)
|
|
state = _state()
|
|
result = await executor.execute(
|
|
"save_note", {"key": "reusable_case", "content": "# 用例\n检查标题"}, "save", state
|
|
)
|
|
assert result.status == "success"
|
|
receipt = json.loads(result.text_summary)
|
|
data = (tmp_path / receipt["path"]).read_bytes()
|
|
assert receipt["verified"] is True and receipt["sha256"] == hashlib.sha256(data).hexdigest()
|
|
assert receipt["bytes"] == len(data)
|
|
read = await executor.execute("read_note", {"key": "reusable_case"}, "read", state)
|
|
assert read.status == "success" and "检查标题" in read.text_summary
|
|
assert result.screenshot_bytes is None and result.ui_elements_text is None
|
|
executor.actuator.execute.assert_not_called()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize("key", ["../outside", "C:\\outside", "output", "output.MD"])
|
|
async def test_flash_note_rejects_paths_and_reserved_final_report(tmp_path, key):
|
|
executor = _make_executor(agent_name="flash")
|
|
executor.ctx.data_engine = Mock(base_dir=tmp_path, current_step_id=None)
|
|
result = await executor.execute("save_note", {"key": key, "content": "data"}, "save", _state())
|
|
assert result.status == "error"
|
|
assert not (tmp_path / "notes").exists()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_flash_note_does_not_fall_back_to_working_directory_without_trace():
|
|
result = await _make_executor(agent_name="flash").execute(
|
|
"save_note", {"key": "report", "content": "data"}, "save", _state()
|
|
)
|
|
assert result.status == "error" and "trace" in result.text_summary
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# ask_explorer
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def test_executor_agent_name_defaults_to_validator_and_is_configurable():
|
|
assert _make_executor().agent_name == "validator"
|
|
assert _make_executor(agent_name="flash").agent_name == "flash"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_ask_explorer_runs_the_pipeline_for_the_executor_agent():
|
|
"""The tier is resolved from the executor's agent name, never from the call."""
|
|
executor = _make_executor(agent_name="flash")
|
|
state = _state()
|
|
outcome = ExplorerOutcome(
|
|
candidates=[ExplorerCandidate(label="S1", coords=(500, 500), description="Send")]
|
|
)
|
|
with patch("artemis.tools.explorer_tool.locate", new=AsyncMock(return_value=outcome)) as locate:
|
|
result = await executor.execute(
|
|
"ask_explorer", {"query": "blue Send button", "context_feedback": "n/a"}, "tc", state
|
|
)
|
|
|
|
locate.assert_awaited_once_with(
|
|
executor.ctx, state, "blue Send button", "n/a", agent_name="flash"
|
|
)
|
|
assert result.status == "success"
|
|
assert "Explorer located 1 candidate(s) for 'blue Send button'" in result.text_summary
|
|
assert "[1] 'Send' at normalized [500, 500]" in result.text_summary
|
|
# This executor's click resolves an index against the element list the
|
|
# candidates just joined, so the guidance teaches the index syntax (Pro
|
|
# parity) next to the coordinates.
|
|
assert "target=3, a bare integer" in result.text_summary
|
|
assert "normalized coordinate" in result.text_summary
|
|
# Candidates were registered on the state for index-based actions.
|
|
assert state.indexed_points == [[540, 1200]]
|
|
assert state.indexed_elements[0]["index"] == 1
|
|
assert result.screenshot_bytes is None
|
|
assert result.ui_elements_text is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_ask_explorer_clean_not_found_is_a_successful_answer():
|
|
executor = _make_executor()
|
|
outcome = ExplorerOutcome(message="Nothing like that is visible; the keyboard covers it.")
|
|
with patch("artemis.tools.explorer_tool.locate", new=AsyncMock(return_value=outcome)):
|
|
result = await executor.execute("ask_explorer", {"query": "gear icon"}, "tc", _state())
|
|
assert result.status == "success"
|
|
assert "Explorer could not locate 'gear icon'" in result.text_summary
|
|
assert "keyboard covers it" in result.text_summary
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_ask_explorer_run_failure_is_an_error_result():
|
|
executor = _make_executor()
|
|
outcome = ExplorerOutcome.failure("Explorer failed: model quota exhausted")
|
|
with patch("artemis.tools.explorer_tool.locate", new=AsyncMock(return_value=outcome)):
|
|
result = await executor.execute("ask_explorer", {"query": "gear icon"}, "tc", _state())
|
|
assert result.status == "error"
|
|
assert "Explorer could not run for 'gear icon'" in result.text_summary
|
|
assert "model quota exhausted" in result.text_summary
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_ask_explorer_accepts_the_legacy_task_description_alias():
|
|
executor = _make_executor()
|
|
outcome = ExplorerOutcome(message="not here")
|
|
with patch("artemis.tools.explorer_tool.locate", new=AsyncMock(return_value=outcome)) as locate:
|
|
result = await executor.execute(
|
|
"ask_explorer", {"task_description": "gear icon top-right"}, "tc", _state()
|
|
)
|
|
assert locate.await_args.args[2] == "gear icon top-right"
|
|
assert locate.await_args.args[3] == ""
|
|
assert result.status == "success"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_ask_explorer_exception_is_contained():
|
|
executor = _make_executor()
|
|
with patch(
|
|
"artemis.tools.explorer_tool.locate", new=AsyncMock(side_effect=RuntimeError("boom"))
|
|
):
|
|
result = await executor.execute("ask_explorer", {"query": "x"}, "tc", _state())
|
|
assert result.status == "error"
|
|
assert result.text_summary == "Error executing ask_explorer: boom"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_default_agent_name_reaches_the_tier_resolver_and_status_follows_the_outcome():
|
|
"""Without an explicit agent name the executor asks the Explorer as ``validator``;
|
|
a clean not-found is an answer, an Explorer run failure is an error."""
|
|
executor = _make_executor()
|
|
assert executor.agent_name == "validator"
|
|
|
|
not_found = ExplorerOutcome(message="Not visible.")
|
|
with patch(
|
|
"artemis.tools.explorer_tool.locate", new=AsyncMock(return_value=not_found)
|
|
) as locate:
|
|
result = await executor.execute(
|
|
"ask_explorer", {"task_description": "gear icon"}, "tc", _state()
|
|
)
|
|
assert locate.await_args.args[2] == "gear icon"
|
|
assert locate.await_args.kwargs["agent_name"] == "validator"
|
|
assert result.status == "success"
|
|
|
|
failed = ExplorerOutcome.failure("Explorer failed: boom")
|
|
with patch("artemis.tools.explorer_tool.locate", new=AsyncMock(return_value=failed)):
|
|
result = await executor.execute("ask_explorer", {"query": "gear icon"}, "tc", _state())
|
|
assert result.status == "error"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_flash_agent_name_reaches_the_tier_resolver():
|
|
"""The Flash profile resolves the Explorer tier from ``agent_name="flash"``."""
|
|
executor = _make_executor(agent_name="flash")
|
|
with patch(
|
|
"artemis.tools.explorer_tool.locate",
|
|
new=AsyncMock(return_value=ExplorerOutcome(message="no")),
|
|
) as locate:
|
|
await executor.execute("ask_explorer", {"query": "x"}, "tc", _state())
|
|
assert locate.await_args.kwargs["agent_name"] == "flash"
|
|
assert locate.await_args.args[0] is executor.ctx
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# video_analyzer
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_video_analyzer_routes_to_the_subagent_as_a_text_tool():
|
|
executor = _make_executor()
|
|
with patch("artemis.agents.video_analyzer.video_analyzer.VideoAnalyzer") as analyzer_cls:
|
|
analyzer_cls.return_value.run = AsyncMock(
|
|
return_value=("The ad finished at 12s; an Error dialog never appeared.", "success")
|
|
)
|
|
result = await executor.execute(
|
|
"video_analyzer",
|
|
{"time_description": "from 5s to 12s", "purpose": "what played"},
|
|
"tc-video",
|
|
None,
|
|
)
|
|
|
|
analyzer_cls.return_value.run.assert_awaited_once_with("from 5s to 12s", "what played")
|
|
assert result.status == "success" # explicit status, not text sniffing
|
|
assert result.text_summary.startswith("The ad finished at 12s")
|
|
assert result.screenshot_bytes is None
|
|
assert result.ui_elements_text is None
|
|
assert result.raw_result is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_video_analyzer_failure_is_an_error_result():
|
|
executor = _make_executor()
|
|
with patch("artemis.agents.video_analyzer.video_analyzer.VideoAnalyzer") as analyzer_cls:
|
|
analyzer_cls.return_value.run = AsyncMock(return_value=("no recording found", "failed"))
|
|
result = await executor.execute(
|
|
"video_analyzer", {"time_description": "from 0s to 3s", "purpose": "x"}, "tc", None
|
|
)
|
|
assert result.status == "error"
|
|
assert result.text_summary == "Video analysis failed: no recording found"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_video_analyzer_exception_is_contained():
|
|
executor = _make_executor()
|
|
with patch("artemis.agents.video_analyzer.video_analyzer.VideoAnalyzer") as analyzer_cls:
|
|
analyzer_cls.return_value.run = AsyncMock(side_effect=RuntimeError("boom"))
|
|
result = await executor.execute(
|
|
"video_analyzer", {"time_description": "from 0s to 3s", "purpose": "x"}, "tc", None
|
|
)
|
|
assert result.status == "error"
|
|
assert "boom" in result.text_summary
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# history tools (search_history / replay_steps / get_step_screenshot)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _history_tool_double(return_value=None, side_effect=None):
|
|
from artemis.tools.history import SearchHistoryArgs
|
|
|
|
tool = Mock()
|
|
tool.args_schema = SearchHistoryArgs
|
|
tool.execute = AsyncMock(return_value=return_value, side_effect=side_effect)
|
|
return tool
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_history_tool_routes_to_the_shared_tool_with_text_result():
|
|
executor = _make_executor()
|
|
tool = _history_tool_double(return_value="- Step 3 (T+00:30): tap btn3 -> executed")
|
|
with patch("artemis.mcp.action_executor.history_tool_by_name", return_value=tool) as by_name:
|
|
result = await executor.execute(
|
|
"search_history",
|
|
{"query": "login Error timeout", "step_range": [1, 5], "unknown_arg": 1},
|
|
"tc-search",
|
|
None,
|
|
)
|
|
|
|
by_name.assert_called_once_with("search_history")
|
|
tool.execute.assert_awaited_once_with(
|
|
ctx=executor.ctx, query="login Error timeout", step_range=[1, 5]
|
|
)
|
|
# A lookup answer mentioning "Error" is still a successful lookup.
|
|
assert result.status == "success"
|
|
assert result.text_summary == "- Step 3 (T+00:30): tap btn3 -> executed"
|
|
assert result.raw_result is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_history_tool_forwards_multimodal_blocks():
|
|
executor = _make_executor()
|
|
blocks = [
|
|
{"type": "text", "text": "Screenshot of step 4 (pre-action) is attached."},
|
|
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,AAA"}},
|
|
]
|
|
tool = _history_tool_double(return_value=blocks)
|
|
with patch("artemis.mcp.action_executor.history_tool_by_name", return_value=tool):
|
|
result = await executor.execute("get_step_screenshot", {"query": "unused"}, "tc", None)
|
|
assert result.status == "success"
|
|
assert result.text_summary == "Screenshot of step 4 (pre-action) is attached."
|
|
assert result.raw_result == blocks
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_history_tool_exception_is_contained():
|
|
executor = _make_executor()
|
|
tool = _history_tool_double(side_effect=RuntimeError("db gone"))
|
|
with patch("artemis.mcp.action_executor.history_tool_by_name", return_value=tool):
|
|
result = await executor.execute("replay_steps", {"query": "x"}, "tc", None)
|
|
# Contained (no exception escapes) and reported structurally: the answer
|
|
# explains the failure and the status says it is one.
|
|
assert result.status == "error"
|
|
assert result.text_summary == "replay_steps failed: db gone"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_history_tools_dispatch_to_the_real_shared_instances():
|
|
"""No per-tool branch: every history tool name resolves through the package."""
|
|
executor = _make_executor() # ctx.data_engine is None -> tools report a failure
|
|
for name in ("search_history", "replay_steps", "get_step_screenshot"):
|
|
result = await executor.execute(
|
|
name, {"query": "x", "start_step": 1, "step_number": 1}, "tc", None
|
|
)
|
|
assert result.status == "error"
|
|
assert (
|
|
"no active execution history" in result.text_summary
|
|
or "no execution history" in result.text_summary
|
|
)
|