"""Helper-tool routing in McpActionExecutor (Flash profile). The Flash runner binds ``ask_explorer``, ``video_analyzer`` and the history tools (``search_history`` / ``replay_steps`` / ``get_step_screenshot``) next to its device actions; the executor must route them to the same implementations the Pro operator's LangChain tools delegate to, as agent-side text tools (no screenshot capture, no device action), and report their status explicitly instead of sniffing the text for the word "Error". """ from types import SimpleNamespace from unittest.mock import AsyncMock, Mock, patch import pytest from artemis.mcp.action_executor import AGENT_TOOL_NAMES, McpActionExecutor from artemis.mcp.action_types import ActionCode, ActionResult from artemis.tools.explorer_tool import ExplorerCandidate, ExplorerOutcome def _make_executor(**kwargs): ctx = Mock() ctx.data_engine = None ctx.device.device_width = 1080 ctx.device.device_height = 2400 actuator = Mock() actuator.controller = Mock() actuator.extensions.return_value = [] return McpActionExecutor(ctx, actuator=actuator, **kwargs) def _state(): state = Mock() state.latest_screenshot = "/tmp/shot.jpg" state.indexed_points = [] state.indexed_elements = [] state.operator_raw_data = None return state def test_helper_tools_are_not_device_actions(): executor = _make_executor() assert { "ask_explorer", "video_analyzer", "search_history", "replay_steps", "get_step_screenshot", } <= AGENT_TOOL_NAMES assert not (AGENT_TOOL_NAMES & executor.action_tool_names) @pytest.mark.asyncio @pytest.mark.parametrize("extension", [False, True]) @pytest.mark.parametrize("ok", [False, True]) async def test_authorized_actions_dispatch_without_secondary_ai_review(extension, ok): from mcp.types import CallToolResult, TextContent executor = _make_executor(agent_name="flash") state = _state() state.initial_goal = ( "仅发送一次指定的问题。等待真实审批表单，Approve once (not remember rule)。" "不要普通聊天代答或重复提交；不向该测试另发消息、不停止任务，也不操作其他会话。" ) name = "submit_form" if extension else "click" args = ( {"window_id": 123} if extension else { "target": [500, 500], "target_description": "Approve", } ) executor.actuator.extensions.return_value = [SimpleNamespace(name="submit_form")] session = SimpleNamespace( call=AsyncMock( return_value=ActionResult( ok=ok, code=ActionCode.OK if ok else ActionCode.DEVICE_ERROR, action=name, message="tool outcome", ) ), call_raw=AsyncMock( return_value=CallToolResult( isError=not ok, content=[TextContent(type="text", text="tool outcome")] ) ), ) executor._session_or_start = AsyncMock(return_value=session) executor._observe_after_action = AsyncMock(return_value=(SimpleNamespace(ok=False), None)) with patch( "artemis.services.llm.get_llm", side_effect=AssertionError("Unexpected AI action review") ) as get_llm: result = await executor.execute(name, args, "action-1", state) get_llm.assert_not_called() wire_args = args if extension else {"target": [500, 500], "times": 1, "delay_ms": 100} dispatched = session.call_raw if extension else session.call unused = session.call if extension else session.call_raw dispatched.assert_awaited_once_with(name, wire_args) unused.assert_not_awaited() assert result.status == ("success" if ok else "error") assert result.text_summary == "tool outcome" @pytest.mark.asyncio async def test_flash_saves_and_reads_text_with_verified_receipt_without_device_actions(tmp_path): import hashlib import json executor = _make_executor(agent_name="flash") executor.ctx.data_engine = Mock(base_dir=tmp_path, current_step_id=None) state = _state() result = await executor.execute( "save_note", {"key": "reusable_case", "content": "# 用例\n检查标题"}, "save", state ) assert result.status == "success" receipt = json.loads(result.text_summary) data = (tmp_path / receipt["path"]).read_bytes() assert receipt["verified"] is True and receipt["sha256"] == hashlib.sha256(data).hexdigest() assert receipt["bytes"] == len(data) read = await executor.execute("read_note", {"key": "reusable_case"}, "read", state) assert read.status == "success" and "检查标题" in read.text_summary assert result.screenshot_bytes is None and result.ui_elements_text is None executor.actuator.execute.assert_not_called() @pytest.mark.asyncio @pytest.mark.parametrize("key", ["../outside", "C:\\outside", "output", "output.MD"]) async def test_flash_note_rejects_paths_and_reserved_final_report(tmp_path, key): executor = _make_executor(agent_name="flash") executor.ctx.data_engine = Mock(base_dir=tmp_path, current_step_id=None) result = await executor.execute("save_note", {"key": key, "content": "data"}, "save", _state()) assert result.status == "error" assert not (tmp_path / "notes").exists() @pytest.mark.asyncio async def test_flash_note_does_not_fall_back_to_working_directory_without_trace(): result = await _make_executor(agent_name="flash").execute( "save_note", {"key": "report", "content": "data"}, "save", _state() ) assert result.status == "error" and "trace" in result.text_summary # --------------------------------------------------------------------------- # ask_explorer # --------------------------------------------------------------------------- def test_executor_agent_name_defaults_to_validator_and_is_configurable(): assert _make_executor().agent_name == "validator" assert _make_executor(agent_name="flash").agent_name == "flash" @pytest.mark.asyncio async def test_ask_explorer_runs_the_pipeline_for_the_executor_agent(): """The tier is resolved from the executor's agent name, never from the call.""" executor = _make_executor(agent_name="flash") state = _state() outcome = ExplorerOutcome( candidates=[ExplorerCandidate(label="S1", coords=(500, 500), description="Send")] ) with patch("artemis.tools.explorer_tool.locate", new=AsyncMock(return_value=outcome)) as locate: result = await executor.execute( "ask_explorer", {"query": "blue Send button", "context_feedback": "n/a"}, "tc", state ) locate.assert_awaited_once_with( executor.ctx, state, "blue Send button", "n/a", agent_name="flash" ) assert result.status == "success" assert "Explorer located 1 candidate(s) for 'blue Send button'" in result.text_summary assert "[1] 'Send' at normalized [500, 500]" in result.text_summary # This executor's click resolves an index against the element list the # candidates just joined, so the guidance teaches the index syntax (Pro # parity) next to the coordinates. assert "target=3, a bare integer" in result.text_summary assert "normalized coordinate" in result.text_summary # Candidates were registered on the state for index-based actions. assert state.indexed_points == [[540, 1200]] assert state.indexed_elements[0]["index"] == 1 assert result.screenshot_bytes is None assert result.ui_elements_text is None @pytest.mark.asyncio async def test_ask_explorer_clean_not_found_is_a_successful_answer(): executor = _make_executor() outcome = ExplorerOutcome(message="Nothing like that is visible; the keyboard covers it.") with patch("artemis.tools.explorer_tool.locate", new=AsyncMock(return_value=outcome)): result = await executor.execute("ask_explorer", {"query": "gear icon"}, "tc", _state()) assert result.status == "success" assert "Explorer could not locate 'gear icon'" in result.text_summary assert "keyboard covers it" in result.text_summary @pytest.mark.asyncio async def test_ask_explorer_run_failure_is_an_error_result(): executor = _make_executor() outcome = ExplorerOutcome.failure("Explorer failed: model quota exhausted") with patch("artemis.tools.explorer_tool.locate", new=AsyncMock(return_value=outcome)): result = await executor.execute("ask_explorer", {"query": "gear icon"}, "tc", _state()) assert result.status == "error" assert "Explorer could not run for 'gear icon'" in result.text_summary assert "model quota exhausted" in result.text_summary @pytest.mark.asyncio async def test_ask_explorer_accepts_the_legacy_task_description_alias(): executor = _make_executor() outcome = ExplorerOutcome(message="not here") with patch("artemis.tools.explorer_tool.locate", new=AsyncMock(return_value=outcome)) as locate: result = await executor.execute( "ask_explorer", {"task_description": "gear icon top-right"}, "tc", _state() ) assert locate.await_args.args[2] == "gear icon top-right" assert locate.await_args.args[3] == "" assert result.status == "success" @pytest.mark.asyncio async def test_ask_explorer_exception_is_contained(): executor = _make_executor() with patch( "artemis.tools.explorer_tool.locate", new=AsyncMock(side_effect=RuntimeError("boom")) ): result = await executor.execute("ask_explorer", {"query": "x"}, "tc", _state()) assert result.status == "error" assert result.text_summary == "Error executing ask_explorer: boom" @pytest.mark.asyncio async def test_default_agent_name_reaches_the_tier_resolver_and_status_follows_the_outcome(): """Without an explicit agent name the executor asks the Explorer as ``validator``; a clean not-found is an answer, an Explorer run failure is an error.""" executor = _make_executor() assert executor.agent_name == "validator" not_found = ExplorerOutcome(message="Not visible.") with patch( "artemis.tools.explorer_tool.locate", new=AsyncMock(return_value=not_found) ) as locate: result = await executor.execute( "ask_explorer", {"task_description": "gear icon"}, "tc", _state() ) assert locate.await_args.args[2] == "gear icon" assert locate.await_args.kwargs["agent_name"] == "validator" assert result.status == "success" failed = ExplorerOutcome.failure("Explorer failed: boom") with patch("artemis.tools.explorer_tool.locate", new=AsyncMock(return_value=failed)): result = await executor.execute("ask_explorer", {"query": "gear icon"}, "tc", _state()) assert result.status == "error" @pytest.mark.asyncio async def test_flash_agent_name_reaches_the_tier_resolver(): """The Flash profile resolves the Explorer tier from ``agent_name="flash"``.""" executor = _make_executor(agent_name="flash") with patch( "artemis.tools.explorer_tool.locate", new=AsyncMock(return_value=ExplorerOutcome(message="no")), ) as locate: await executor.execute("ask_explorer", {"query": "x"}, "tc", _state()) assert locate.await_args.kwargs["agent_name"] == "flash" assert locate.await_args.args[0] is executor.ctx # --------------------------------------------------------------------------- # video_analyzer # --------------------------------------------------------------------------- @pytest.mark.asyncio async def test_video_analyzer_routes_to_the_subagent_as_a_text_tool(): executor = _make_executor() with patch("artemis.agents.video_analyzer.video_analyzer.VideoAnalyzer") as analyzer_cls: analyzer_cls.return_value.run = AsyncMock( return_value=("The ad finished at 12s; an Error dialog never appeared.", "success") ) result = await executor.execute( "video_analyzer", {"time_description": "from 5s to 12s", "purpose": "what played"}, "tc-video", None, ) analyzer_cls.return_value.run.assert_awaited_once_with("from 5s to 12s", "what played") assert result.status == "success" # explicit status, not text sniffing assert result.text_summary.startswith("The ad finished at 12s") assert result.screenshot_bytes is None assert result.ui_elements_text is None assert result.raw_result is None @pytest.mark.asyncio async def test_video_analyzer_failure_is_an_error_result(): executor = _make_executor() with patch("artemis.agents.video_analyzer.video_analyzer.VideoAnalyzer") as analyzer_cls: analyzer_cls.return_value.run = AsyncMock(return_value=("no recording found", "failed")) result = await executor.execute( "video_analyzer", {"time_description": "from 0s to 3s", "purpose": "x"}, "tc", None ) assert result.status == "error" assert result.text_summary == "Video analysis failed: no recording found" @pytest.mark.asyncio async def test_video_analyzer_exception_is_contained(): executor = _make_executor() with patch("artemis.agents.video_analyzer.video_analyzer.VideoAnalyzer") as analyzer_cls: analyzer_cls.return_value.run = AsyncMock(side_effect=RuntimeError("boom")) result = await executor.execute( "video_analyzer", {"time_description": "from 0s to 3s", "purpose": "x"}, "tc", None ) assert result.status == "error" assert "boom" in result.text_summary # --------------------------------------------------------------------------- # history tools (search_history / replay_steps / get_step_screenshot) # --------------------------------------------------------------------------- def _history_tool_double(return_value=None, side_effect=None): from artemis.tools.history import SearchHistoryArgs tool = Mock() tool.args_schema = SearchHistoryArgs tool.execute = AsyncMock(return_value=return_value, side_effect=side_effect) return tool @pytest.mark.asyncio async def test_history_tool_routes_to_the_shared_tool_with_text_result(): executor = _make_executor() tool = _history_tool_double(return_value="- Step 3 (T+00:30): tap btn3 -> executed") with patch("artemis.mcp.action_executor.history_tool_by_name", return_value=tool) as by_name: result = await executor.execute( "search_history", {"query": "login Error timeout", "step_range": [1, 5], "unknown_arg": 1}, "tc-search", None, ) by_name.assert_called_once_with("search_history") tool.execute.assert_awaited_once_with( ctx=executor.ctx, query="login Error timeout", step_range=[1, 5] ) # A lookup answer mentioning "Error" is still a successful lookup. assert result.status == "success" assert result.text_summary == "- Step 3 (T+00:30): tap btn3 -> executed" assert result.raw_result is None @pytest.mark.asyncio async def test_history_tool_forwards_multimodal_blocks(): executor = _make_executor() blocks = [ {"type": "text", "text": "Screenshot of step 4 (pre-action) is attached."}, {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,AAA"}}, ] tool = _history_tool_double(return_value=blocks) with patch("artemis.mcp.action_executor.history_tool_by_name", return_value=tool): result = await executor.execute("get_step_screenshot", {"query": "unused"}, "tc", None) assert result.status == "success" assert result.text_summary == "Screenshot of step 4 (pre-action) is attached." assert result.raw_result == blocks @pytest.mark.asyncio async def test_history_tool_exception_is_contained(): executor = _make_executor() tool = _history_tool_double(side_effect=RuntimeError("db gone")) with patch("artemis.mcp.action_executor.history_tool_by_name", return_value=tool): result = await executor.execute("replay_steps", {"query": "x"}, "tc", None) # Contained (no exception escapes) and reported structurally: the answer # explains the failure and the status says it is one. assert result.status == "error" assert result.text_summary == "replay_steps failed: db gone" @pytest.mark.asyncio async def test_history_tools_dispatch_to_the_real_shared_instances(): """No per-tool branch: every history tool name resolves through the package.""" executor = _make_executor() # ctx.data_engine is None -> tools report a failure for name in ("search_history", "replay_steps", "get_step_screenshot"): result = await executor.execute( name, {"query": "x", "start_step": 1, "step_number": 1}, "tc", None ) assert result.status == "error" assert ( "no active execution history" in result.text_summary or "no execution history" in result.text_summary )