feat: extract in-process MCP action layer, concurrent task dispatch, and spawn/reconcile hardening
- Extract device action execution into artemis/mcp action layer: typed ActionResult, capability manifest with startup validation, AdbActuator as the single physical implementation, in-memory ActionSession (replaces stdio subprocess MCP in validator), and McpActionExecutor preserving the legacy executor contract. adb_server keeps its external stdio/SSE surface unchanged and now reuses the actuator implementation; import-time side effects moved into configure_stdio_mode(). - Capability-gated prompt assembly (prompt_assembly.py): tool enums and teaching segments render per available backend tools; full-set output is byte-identical to the previous static text (locked by tests). - Admin console task queue: serial worker replaced by a per-device concurrent dispatcher (ARTEMIS_CONCURRENCY_MODE=global restores serial semantics), per-session cancellation, strict device binding on enqueue, unified stdio piping with tee to trace stdout.log, and shutdown cleanup of all active runs. Fix _pid_exists exception order so PermissionError is treated as alive. - mcp_server reliability: spawn watchdog kills runners stuck before interpreter startup and releases device reservations; task state reconcile uses psutil liveness, prefers runner-reported pids, and adds a 45s startup grace to stop false failure reports on Windows; conversation_id is now optional end-to-end; explicit device_serial is validated at submission. - CLI install now writes ~/.claude/rules/artemis.md for Claude Code and migrates/removes the legacy CLAUDE.md injection block; ~/.claude.json is never rewritten when it fails to parse. - Tests: Windows-portable stdio lifecycle and path assertions; new suites for action layer, prompt assembly, spawn watchdog, and task reconcile (895 passed, 3 skipped).
This commit is contained in:
1 parent
60312abbb0
commit
cbdca1497c
52 files changed
+5735
-1649
No files matched your search
@@ -304,9 +304,11 @@ async def test_input_text_happy_path(mock_context, mock_controller):
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Executed typing 'hello'." in outcome
|
||||
mock_controller.tap_at.assert_called_once_with(540, 1200)
|
||||
# Focus goes through ensure_focus_at_coords (keyword-style tap), and typing
|
||||
# never re-clears: the clear already happened via erase_text.
|
||||
mock_controller.tap_at.assert_called_once_with(x=540, y=1200)
|
||||
mock_controller.erase_text.assert_called_once()
|
||||
mock_controller.type_text.assert_called_once_with("hello")
|
||||
mock_controller.type_text.assert_called_once_with("hello", clear_existing=False)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -396,8 +398,10 @@ async def test_press_key_input_validation(mock_context, mock_controller):
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
# The current execution doesn't raise error on unknown keys, just logs error
|
||||
assert "Error executing key press 'INVALID_KEY'." in outcome
|
||||
# Unknown keycodes are forwarded verbatim to the driver (historical adb_server
|
||||
# behavior); the mock driver accepts them, so the press reports success.
|
||||
assert "Executed key press 'INVALID_KEY'." in outcome
|
||||
mock_controller.press_key.assert_called_once_with("INVALID_KEY")
|
||||
|
||||
|
||||
# ==========================================
|
||||
@@ -409,8 +413,10 @@ async def test_press_key_input_validation(mock_context, mock_controller):
|
||||
async def test_manage_app_happy_path(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
with (
|
||||
patch("artemis.agents.validator.tool_declarations.find_package") as mock_find_package,
|
||||
patch("artemis.agents.validator.tool_declarations.launch_app_with_retries") as mock_launch,
|
||||
# The device-call bodies moved into the actuator layer, which imports these
|
||||
# lazily from their source modules -- patch them at the source.
|
||||
patch("artemis.tools.mobile.launch_app.find_package") as mock_find_package,
|
||||
patch("artemis.utils.app_launch_utils.launch_app_with_retries") as mock_launch,
|
||||
):
|
||||
mock_find_package.return_value = "com.google.android.youtube"
|
||||
mock_launch.return_value = (True, "")
|
||||
@@ -430,7 +436,7 @@ async def test_manage_app_happy_path(mock_context, mock_controller):
|
||||
@pytest.mark.asyncio
|
||||
async def test_manage_app_input_validation(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
with patch("artemis.agents.validator.tool_declarations.find_package") as mock_find_package:
|
||||
with patch("artemis.tools.mobile.launch_app.find_package") as mock_find_package:
|
||||
mock_find_package.return_value = None
|
||||
|
||||
outcome, _, _, _ = await _exec_manage_app(
|
||||
|
||||
@@ -0,0 +1,233 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Prompt assembly tests: full-set output is byte-stable, absent tools leave no trace."""
|
||||
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
from jinja2 import Template
|
||||
import pytest
|
||||
|
||||
from artemis.agents.operator.prompt_builder import load_operator_prompts
|
||||
from artemis.agents.operator.prompts import (
|
||||
OPERATOR_PROMPT_TOOLSET,
|
||||
apply_operator_prompt_contract,
|
||||
)
|
||||
from artemis.agents.prompt_assembly import gate_segment, render_tool_enum
|
||||
from artemis.agents.validator.failure_analyzer import (
|
||||
FailureAnalysisStrategy,
|
||||
FailureAnalyzer,
|
||||
PixelTargetDisappearedStrategy,
|
||||
TargetDisappearedStrategy,
|
||||
)
|
||||
from artemis.agents.validator.validator import ValidationErrorCategory
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
|
||||
OPERATOR_DEVICE_TOOLS = (
|
||||
"click",
|
||||
"input_text",
|
||||
"swipe",
|
||||
"press_key",
|
||||
"manage_app",
|
||||
"wait_for_delay",
|
||||
"long_press",
|
||||
)
|
||||
|
||||
|
||||
def _tool_refs(text: str, tool: str) -> list[str]:
|
||||
"""Finds real tool references (backticked or call form), ignoring prose."""
|
||||
return re.findall(rf"(`{tool}`|\b{tool}\()", text)
|
||||
|
||||
|
||||
# --- render_tool_enum ----------------------------------------------------------------
|
||||
|
||||
|
||||
def test_enum_plain_list():
|
||||
out = render_tool_enum(("a", "b", "c"), {"a", "b", "c"})
|
||||
assert out == "`a`, `b`, `c`"
|
||||
|
||||
|
||||
def test_enum_oxford_or_and():
|
||||
tools = ("a", "b", "c")
|
||||
assert render_tool_enum(tools, set(tools), final_sep="or") == "`a`, `b`, or `c`"
|
||||
assert render_tool_enum(tools, set(tools), final_sep="and") == "`a`, `b`, and `c`"
|
||||
|
||||
|
||||
def test_enum_two_items_no_oxford_comma():
|
||||
assert render_tool_enum(("a", "b"), {"a", "b"}, final_sep="or") == "`a` or `b`"
|
||||
|
||||
|
||||
def test_enum_single_and_empty():
|
||||
assert render_tool_enum(("a", "b"), {"a"}) == "`a`"
|
||||
assert render_tool_enum(("a", "b"), set()) == ""
|
||||
|
||||
|
||||
def test_gate_segment():
|
||||
assert gate_segment("teach", {"swipe"}, "swipe") == "teach"
|
||||
assert gate_segment("teach", set(), "swipe") == ""
|
||||
assert gate_segment("teach", {"a"}, "a", "b") == ""
|
||||
|
||||
|
||||
# --- Operator prompt assembly --------------------------------------------------------
|
||||
|
||||
|
||||
def test_full_toolset_is_the_default():
|
||||
"""``available_tools=None`` must equal the explicit full set, byte for byte."""
|
||||
for template in load_operator_prompts().values():
|
||||
assert apply_operator_prompt_contract(template) == apply_operator_prompt_contract(
|
||||
template, available_tools=OPERATOR_PROMPT_TOOLSET
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tool", OPERATOR_DEVICE_TOOLS + ("video_analyzer",))
|
||||
def test_removed_tool_leaves_no_reference(tool):
|
||||
"""The executable definition of 'an absent tool costs the model nothing'."""
|
||||
for name, template in load_operator_prompts().items():
|
||||
out = apply_operator_prompt_contract(
|
||||
template, available_tools=OPERATOR_PROMPT_TOOLSET - {tool}
|
||||
)
|
||||
assert not _tool_refs(out, tool), f"'{tool}' still referenced in {name}"
|
||||
|
||||
|
||||
def test_removed_swipe_takes_its_teaching_along():
|
||||
"""Gating removes the drag/slider teaching, not just the backticked name."""
|
||||
template = load_operator_prompts()["main_template"]
|
||||
out = apply_operator_prompt_contract(
|
||||
template, available_tools=OPERATOR_PROMPT_TOOLSET - {"swipe"}
|
||||
)
|
||||
assert "swipe/drag" not in out
|
||||
assert "sliders/SeekBars" not in out
|
||||
# The generic gesture-scope sentences survive.
|
||||
assert "Action Scope & Interaction Precision" in out
|
||||
|
||||
|
||||
def test_removed_wait_for_delay_keeps_interval_line_coherent():
|
||||
"""Only the wait_for_delay parenthetical goes; the SOP grammar line stays."""
|
||||
template = load_operator_prompts()["main_template"]
|
||||
out = apply_operator_prompt_contract(
|
||||
template, available_tools=OPERATOR_PROMPT_TOOLSET - {"wait_for_delay"}
|
||||
)
|
||||
assert "`Interval: <Interval>`: Declared polling cadence." in out
|
||||
assert "Loading & Transitions" not in out
|
||||
|
||||
|
||||
def test_reduced_enums_stay_well_formed():
|
||||
template = load_operator_prompts()["main_template"]
|
||||
out = apply_operator_prompt_contract(
|
||||
template,
|
||||
available_tools=OPERATOR_PROMPT_TOOLSET - {"manage_app", "wait_for_delay"},
|
||||
)
|
||||
assert (
|
||||
"Physical device actions (`click`, `input_text`, `swipe`, `press_key`,"
|
||||
" `long_press`)" in out
|
||||
)
|
||||
assert (
|
||||
"Turn-Ending Action (`click`, `swipe`, `input_text`, `long_press`, or"
|
||||
" `press_key`)" in out
|
||||
)
|
||||
|
||||
|
||||
def test_full_set_enum_slots_render_verbatim():
|
||||
"""With every tool present the historical enumeration wording is reproduced."""
|
||||
template = load_operator_prompts()["main_template"]
|
||||
out = apply_operator_prompt_contract(template)
|
||||
assert (
|
||||
"Physical device actions (`click`, `input_text`, `swipe`, `press_key`,"
|
||||
" `manage_app`, `wait_for_delay`, `long_press`)" in out
|
||||
)
|
||||
assert (
|
||||
"Turn-Ending Action (`click`, `swipe`, `input_text`, `long_press`,"
|
||||
" `press_key`, `manage_app`, or `wait_for_delay`)" in out
|
||||
)
|
||||
assert (
|
||||
"(`ask_explorer`, `ask_diagnoser`, `video_analyzer`, `run_adb_command`,"
|
||||
" `manage_task`, `analyze_task_output`, `read_note`, `list_notes`, and"
|
||||
" `save_note`)" in out
|
||||
)
|
||||
|
||||
|
||||
# --- Flash prompt assembly -----------------------------------------------------------
|
||||
|
||||
|
||||
def _render_flash(available_tools: frozenset[str]) -> str:
|
||||
template_text = (REPO_ROOT / "artemis/agents/flash/flash_runner.md").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
return Template(template_text).render(goal="test goal", available_tools=available_tools)
|
||||
|
||||
|
||||
FLASH_FULL = frozenset({"manage_app", "wait_for_delay", "click_sequence", "click", "swipe"})
|
||||
|
||||
|
||||
def test_flash_full_set_keeps_both_segments():
|
||||
out = _render_flash(FLASH_FULL)
|
||||
assert "**App Launching**" in out
|
||||
assert "**Timed Waiting & Transitions**" in out
|
||||
assert "{%" not in out
|
||||
|
||||
|
||||
def test_flash_without_manage_app_drops_app_launching():
|
||||
out = _render_flash(FLASH_FULL - {"manage_app"})
|
||||
assert "**App Launching**" not in out
|
||||
assert not _tool_refs(out, "manage_app")
|
||||
# The neighbouring numbered rule survives.
|
||||
assert "**Screenshot Pruning Awareness**" in out
|
||||
|
||||
|
||||
def test_flash_without_wait_for_delay_drops_timed_waiting():
|
||||
out = _render_flash(FLASH_FULL - {"wait_for_delay"})
|
||||
assert "**Timed Waiting & Transitions**" not in out
|
||||
assert not _tool_refs(out, "wait_for_delay")
|
||||
assert "**State Drift & Progression Tolerance**" in out
|
||||
|
||||
|
||||
# --- FailureAnalyzer strategy gating -------------------------------------------------
|
||||
|
||||
|
||||
def _analyzer() -> FailureAnalyzer:
|
||||
return FailureAnalyzer.__new__(FailureAnalyzer) # _select_strategy needs no ctx
|
||||
|
||||
|
||||
def test_strategy_selection_unchanged_without_gating():
|
||||
analyzer = _analyzer()
|
||||
assert isinstance(
|
||||
analyzer._select_strategy(ValidationErrorCategory.TARGET_DISAPPEARED),
|
||||
TargetDisappearedStrategy,
|
||||
)
|
||||
assert isinstance(
|
||||
analyzer._select_strategy(ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED),
|
||||
PixelTargetDisappearedStrategy,
|
||||
)
|
||||
|
||||
|
||||
def test_strategy_falls_back_when_required_tool_missing():
|
||||
"""Strategy + prompt are one unit: no click_sequence, no disappeared-prompts."""
|
||||
analyzer = _analyzer()
|
||||
strategy = analyzer._select_strategy(
|
||||
ValidationErrorCategory.TARGET_DISAPPEARED,
|
||||
available_tools=frozenset({"click", "swipe"}),
|
||||
)
|
||||
assert type(strategy) is FailureAnalysisStrategy
|
||||
assert strategy.get_prompt_template_name() == "failure_analyzer.md"
|
||||
|
||||
|
||||
def test_strategy_kept_when_required_tool_present():
|
||||
analyzer = _analyzer()
|
||||
strategy = analyzer._select_strategy(
|
||||
ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED,
|
||||
available_tools=frozenset({"click_sequence"}),
|
||||
)
|
||||
assert isinstance(strategy, PixelTargetDisappearedStrategy)
|
||||
+112
-251
@@ -12,13 +12,25 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""ValidatorNode tests over the in-process unified action session.
|
||||
|
||||
The Validator no longer spawns a stdio MCP subprocess; it calls a shared
|
||||
``ActionSession`` whose device actions return structured ``ActionResult`` objects.
|
||||
``FakeActionSession`` below stands in for it: ``action_handler`` decides each
|
||||
action's ``(ok, message)``, ``screenshot`` feeds the polling loop, and ``hierarchy``
|
||||
feeds the safety-net XML validation.
|
||||
"""
|
||||
|
||||
import json
|
||||
from unittest.mock import AsyncMock, MagicMock, Mock, patch
|
||||
|
||||
from langchain_core.messages import AIMessage
|
||||
import pytest
|
||||
|
||||
from artemis.agents.validator.failure_analyzer import ValidationErrorCategory
|
||||
from artemis.agents.validator.validator import ValidatorNode
|
||||
from artemis.context import ArtemisContext
|
||||
import pytest
|
||||
from artemis.mcp.action_types import ActionCode, ActionResult
|
||||
|
||||
|
||||
class DummyState:
|
||||
@@ -38,6 +50,46 @@ class DummyState:
|
||||
return update
|
||||
|
||||
|
||||
class FakeActionSession:
|
||||
"""Test double for artemis.mcp.action_session.ActionSession."""
|
||||
|
||||
def __init__(self):
|
||||
self.started = True
|
||||
self.calls: list[tuple[str, dict]] = []
|
||||
# (name, args) -> (ok, message)
|
||||
self.action_handler = lambda name, args: (True, "")
|
||||
# str | callable | Exception
|
||||
self.screenshot = "ZHVtbXlfZGF0YQ==" # b64("dummy_data")
|
||||
self.hierarchy: list | Exception = []
|
||||
|
||||
async def call(self, name, args):
|
||||
self.calls.append((name, args))
|
||||
ok, msg = self.action_handler(name, args)
|
||||
return ActionResult(
|
||||
ok=ok,
|
||||
code=ActionCode.OK if ok else ActionCode.DEVICE_ERROR,
|
||||
action=name,
|
||||
message=msg,
|
||||
)
|
||||
|
||||
async def screenshot_b64(self, timeout=None):
|
||||
shot = self.screenshot
|
||||
if isinstance(shot, Exception):
|
||||
raise shot
|
||||
return shot() if callable(shot) else shot
|
||||
|
||||
async def ui_hierarchy(self, timeout=None):
|
||||
if isinstance(self.hierarchy, Exception):
|
||||
raise self.hierarchy
|
||||
return self.hierarchy
|
||||
|
||||
async def aclose(self):
|
||||
pass
|
||||
|
||||
def calls_for(self, name: str) -> list[dict]:
|
||||
return [args for n, args in self.calls if n == name]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_context(tmp_path):
|
||||
ctx = Mock(spec=ArtemisContext)
|
||||
@@ -45,6 +97,9 @@ def mock_context(tmp_path):
|
||||
ctx.data_engine = Mock()
|
||||
ctx.data_engine.base_dir = tmp_path
|
||||
ctx.data_engine.get_relative_time.return_value = 1.0
|
||||
ctx.device = Mock()
|
||||
ctx.device.device_width = 1080
|
||||
ctx.device.device_height = 2400
|
||||
return ctx
|
||||
|
||||
|
||||
@@ -57,31 +112,18 @@ def temp_screenshot(tmp_path):
|
||||
|
||||
@pytest.fixture
|
||||
def mock_mcp():
|
||||
with (
|
||||
patch("mcp.client.stdio.stdio_client") as mock_stdio,
|
||||
patch("artemis.agents.validator.validator.ClientSession") as mock_session_cls,
|
||||
):
|
||||
mock_ctx = AsyncMock()
|
||||
mock_ctx.__aenter__.return_value = (Mock(), Mock())
|
||||
mock_stdio.return_value = mock_ctx
|
||||
fake = FakeActionSession()
|
||||
|
||||
mock_session = AsyncMock()
|
||||
mock_session.initialize = AsyncMock()
|
||||
mock_session_cls.return_value = mock_session
|
||||
async def _get_session(ctx, actuator=None):
|
||||
return fake
|
||||
|
||||
yield mock_session
|
||||
with patch("artemis.agents.validator.validator.get_action_session", _get_session):
|
||||
yield fake
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_success(mock_mcp, mock_context, temp_screenshot):
|
||||
"""Test that ValidatorNode executes actions successfully."""
|
||||
# Mock MCP calls: screenshot, tap, screenshot
|
||||
mock_mcp.call_tool.side_effect = [
|
||||
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # pre-screenshot
|
||||
Mock(content=[Mock(text="Success")]), # tap action
|
||||
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # post-screenshot
|
||||
]
|
||||
|
||||
decisions = json.dumps([{"action": "tap", "coordinates": [105, 205]}])
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
|
||||
@@ -95,78 +137,23 @@ async def test_validator_success(mock_mcp, mock_context, temp_screenshot):
|
||||
assert len(report["execution"]) == 1
|
||||
assert report["execution"][0]["action"] == "tap"
|
||||
assert "attempts" not in report["execution"][0]
|
||||
# The Operator's tap verb was translated to the canonical click tool.
|
||||
assert len(mock_mcp.calls_for("click")) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failed_mcp_handshake_exits_contexts_and_disables_child_awake_policy():
|
||||
"""A handshake failure must not leave AnyIO cancellation scopes behind."""
|
||||
events = []
|
||||
|
||||
class FakeClientContext:
|
||||
async def __aenter__(self):
|
||||
events.append("client_enter")
|
||||
return Mock(), Mock()
|
||||
|
||||
async def __aexit__(self, exc_type, exc, traceback):
|
||||
events.append("client_exit")
|
||||
|
||||
class FakeSession:
|
||||
async def __aenter__(self):
|
||||
events.append("session_enter")
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc, traceback):
|
||||
events.append("session_exit")
|
||||
|
||||
async def initialize(self):
|
||||
raise TimeoutError("handshake timed out")
|
||||
|
||||
ctx = Mock(spec=ArtemisContext)
|
||||
ctx.mcp_client_ctx = None
|
||||
ctx.mcp_session = None
|
||||
fake_client_ctx = FakeClientContext()
|
||||
fake_session = FakeSession()
|
||||
captured_parameters = []
|
||||
|
||||
def fake_stdio_client(parameters):
|
||||
captured_parameters.append(parameters)
|
||||
return fake_client_ctx
|
||||
|
||||
node = ValidatorNode(ctx)
|
||||
with (
|
||||
patch("artemis.agents.validator.validator.stdio_client", fake_stdio_client),
|
||||
patch("artemis.agents.validator.validator.ClientSession", return_value=fake_session),
|
||||
patch("artemis.agents.validator.validator.AdbClient") as adb_client,
|
||||
patch.object(node, "_kill_mcp_server_instantly") as emergency_kill,
|
||||
):
|
||||
adb_client.return_value.device_list.return_value = []
|
||||
result = await node._get_mcp_session()
|
||||
|
||||
assert result is None
|
||||
assert events == ["client_enter", "session_enter", "session_exit", "client_exit"]
|
||||
assert ctx.mcp_session is None
|
||||
assert ctx.mcp_client_ctx is None
|
||||
assert captured_parameters[0].env["ARTEMIS_KEEP_DEVICE_AWAKE"] == "false"
|
||||
emergency_kill.assert_not_called()
|
||||
|
||||
# NOTE: upstream's test_failed_mcp_handshake_exits_contexts_and_disables_child_awake_policy
|
||||
# guarded the stdio-subprocess handshake teardown, machinery this migration removed
|
||||
# entirely. Its hazard class (leaked AnyIO cancel scopes) is now pinned by
|
||||
# tests/unit/mcp/test_action_session.py against the in-process ActionSession.
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_failure_analysis(mock_mcp, mock_context, temp_screenshot):
|
||||
"""Test that ValidatorNode triggers failure analysis on error and handles
|
||||
cannot_fix via tool.
|
||||
"""
|
||||
tap_count = 0
|
||||
|
||||
def mock_call_tool(name, args):
|
||||
nonlocal tap_count
|
||||
if name == "take_screenshot":
|
||||
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
|
||||
if name == "tap":
|
||||
tap_count += 1
|
||||
return Mock(content=[Mock(text="Error: Element not found")])
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
|
||||
mock_mcp.call_tool.side_effect = mock_call_tool
|
||||
mock_mcp.action_handler = lambda name, args: (
|
||||
(False, "Error: Element not found") if name == "click" else (True, "")
|
||||
)
|
||||
|
||||
decisions = json.dumps([{"action": "tap", "coordinates": [105, 205]}])
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
@@ -203,18 +190,15 @@ async def test_validator_repair_success(mock_mcp, mock_context, temp_screenshot)
|
||||
"""
|
||||
tap_count = 0
|
||||
|
||||
def mock_call_tool(name, args):
|
||||
def handler(name, args):
|
||||
nonlocal tap_count
|
||||
if name == "take_screenshot":
|
||||
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
|
||||
if name == "tap":
|
||||
if name == "click":
|
||||
tap_count += 1
|
||||
if tap_count <= 2: # First tap fails
|
||||
return Mock(content=[Mock(text="Error: Element not found")])
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
if tap_count <= 2: # First action fails on both validator attempts
|
||||
return False, "Error: Element not found"
|
||||
return True, ""
|
||||
|
||||
mock_mcp.call_tool.side_effect = mock_call_tool
|
||||
mock_mcp.action_handler = handler
|
||||
|
||||
# Two actions planned initially
|
||||
decisions = json.dumps(
|
||||
@@ -254,12 +238,6 @@ async def test_validator_repair_success(mock_mcp, mock_context, temp_screenshot)
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_wait_for_delay(mock_mcp, mock_context, temp_screenshot):
|
||||
"""Test that wait_for_delay executes successfully."""
|
||||
|
||||
def mock_call(name, args):
|
||||
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
|
||||
|
||||
mock_mcp.call_tool.side_effect = mock_call
|
||||
|
||||
decisions = json.dumps([{"action": "wait_for_delay", "time_in_ms": 10}])
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
|
||||
@@ -272,19 +250,13 @@ async def test_validator_wait_for_delay(mock_mcp, mock_context, temp_screenshot)
|
||||
assert "execution" in report
|
||||
assert len(report["execution"]) == 1
|
||||
assert report["execution"][0]["action"] == "wait_for_delay"
|
||||
# wait_for_delay is a pure client-side sleep, never a session call.
|
||||
assert mock_mcp.calls_for("wait_for_delay") == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_focus_and_clear_text_no_ui_change(mock_mcp, mock_context, temp_screenshot):
|
||||
"""Test that focus_and_clear_text executes successfully even if no UI change is detected."""
|
||||
|
||||
def mock_call(name, args):
|
||||
if name == "focus_and_clear_text":
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
|
||||
|
||||
mock_mcp.call_tool.side_effect = mock_call
|
||||
|
||||
decisions = json.dumps([{"action": "focus_and_clear_text", "coordinates": [400, 500]}])
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
|
||||
@@ -297,6 +269,7 @@ async def test_validator_focus_and_clear_text_no_ui_change(mock_mcp, mock_contex
|
||||
assert "execution" in report
|
||||
assert len(report["execution"]) == 1
|
||||
assert report["execution"][0]["action"] == "focus_and_clear_text"
|
||||
assert len(mock_mcp.calls_for("focus_and_clear_text")) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -304,16 +277,6 @@ async def test_validator_silent_failure_treated_as_success(mock_mcp, mock_contex
|
||||
"""Test that silent failure (exec succeeds but no UI change) is treated as
|
||||
success and does not trigger FailureAnalyzer.
|
||||
"""
|
||||
|
||||
def mock_call_tool(name, args):
|
||||
if name == "take_screenshot":
|
||||
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
|
||||
if name == "tap":
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
|
||||
mock_mcp.call_tool.side_effect = mock_call_tool
|
||||
|
||||
decisions = json.dumps([{"action": "tap", "coordinates": [105, 205]}])
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
|
||||
@@ -344,13 +307,7 @@ async def test_validator_records_skipped_actions_on_cannot_fix(
|
||||
mock_mcp, mock_context, temp_screenshot
|
||||
):
|
||||
"""Test that ValidatorNode marks unexecuted actions as Skipped when repair fails."""
|
||||
|
||||
def mock_call_tool(name, args):
|
||||
if name == "take_screenshot":
|
||||
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
|
||||
return Mock(content=[Mock(text="Error: Connection lost")])
|
||||
|
||||
mock_mcp.call_tool.side_effect = mock_call_tool
|
||||
mock_mcp.action_handler = lambda name, args: (False, "Error: Connection lost")
|
||||
|
||||
# Two actions planned
|
||||
decisions = json.dumps(
|
||||
@@ -399,7 +356,7 @@ async def test_validator_pre_execution_validation_and_repair(
|
||||
"""Test that pre-execution validation fails when element changes, and is
|
||||
repaired by FailureAnalyzer.
|
||||
"""
|
||||
mock_live_elements = [
|
||||
mock_mcp.hierarchy = [
|
||||
{
|
||||
"text": "Sign Up",
|
||||
"bounds": "[100,200][300,300]",
|
||||
@@ -407,17 +364,6 @@ async def test_validator_pre_execution_validation_and_repair(
|
||||
}
|
||||
]
|
||||
|
||||
def mock_call_tool(name, args):
|
||||
if name == "take_screenshot":
|
||||
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
|
||||
if name == "get_ui_hierarchy":
|
||||
return Mock(content=[Mock(text=json.dumps(mock_live_elements))])
|
||||
if name == "tap":
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
|
||||
mock_mcp.call_tool.side_effect = mock_call_tool
|
||||
|
||||
decisions = json.dumps(
|
||||
[
|
||||
{
|
||||
@@ -477,7 +423,7 @@ async def test_validator_pre_execution_validation_self_healing(
|
||||
mock_mcp, mock_context, temp_screenshot
|
||||
):
|
||||
"""Test that pre-execution validation succeeds and self-heals when element shifts slightly."""
|
||||
mock_live_elements = [
|
||||
mock_mcp.hierarchy = [
|
||||
{
|
||||
"text": "Login",
|
||||
"bounds": "[100,220][300,320]",
|
||||
@@ -485,21 +431,6 @@ async def test_validator_pre_execution_validation_self_healing(
|
||||
}
|
||||
]
|
||||
|
||||
tapped_coordinates = None
|
||||
|
||||
def mock_call_tool(name, args):
|
||||
nonlocal tapped_coordinates
|
||||
if name == "take_screenshot":
|
||||
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
|
||||
if name == "get_ui_hierarchy":
|
||||
return Mock(content=[Mock(text=json.dumps(mock_live_elements))])
|
||||
if name == "tap":
|
||||
tapped_coordinates = args.get("coordinates")
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
|
||||
mock_mcp.call_tool.side_effect = mock_call_tool
|
||||
|
||||
decisions = json.dumps(
|
||||
[
|
||||
{
|
||||
@@ -518,8 +449,12 @@ async def test_validator_pre_execution_validation_self_healing(
|
||||
with patch("artemis.utils.image_diff.check_ui_change", return_value=True):
|
||||
await node(state)
|
||||
|
||||
# Center of [100, 220][300, 320] is [200, 270]
|
||||
assert tapped_coordinates == [200, 270]
|
||||
# Center of [100, 220][300, 320] is pixel [200, 270]; the healed coordinates
|
||||
# travel to the canonical click tool normalized to 0-1000 on a 1080x2400 screen:
|
||||
# round(200 * 1000 / 1080) = 185, round(270 * 1000 / 2400) = 112.
|
||||
clicks = mock_mcp.calls_for("click")
|
||||
assert len(clicks) == 1
|
||||
assert clicks[0]["target"] == [185, 112]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -531,7 +466,7 @@ async def test_validator_pre_execution_validation_anonymous_occupant(
|
||||
"""
|
||||
# The live screen contains an anonymous clickable view covering
|
||||
# target coordinates [200, 250]
|
||||
mock_live_elements = [
|
||||
mock_mcp.hierarchy = [
|
||||
{
|
||||
"text": "",
|
||||
"bounds": "[150,200][250,300]",
|
||||
@@ -540,15 +475,6 @@ async def test_validator_pre_execution_validation_anonymous_occupant(
|
||||
}
|
||||
]
|
||||
|
||||
def mock_call_tool(name, args):
|
||||
if name == "take_screenshot":
|
||||
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
|
||||
if name == "get_ui_hierarchy":
|
||||
return Mock(content=[Mock(text=json.dumps(mock_live_elements))])
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
|
||||
mock_mcp.call_tool.side_effect = mock_call_tool
|
||||
|
||||
decisions = json.dumps(
|
||||
[
|
||||
{
|
||||
@@ -596,12 +522,6 @@ async def test_validator_pre_execution_validation_anonymous_occupant(
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_pixel_validation_success(mock_mcp, mock_context, temp_screenshot):
|
||||
"""Test that the pixel safety net succeeds when Gemini reports target is present."""
|
||||
mock_mcp.call_tool.side_effect = [
|
||||
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # live screenshot for pixel check
|
||||
Mock(content=[Mock(text="Success")]), # tap action
|
||||
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # post-screenshot
|
||||
]
|
||||
|
||||
decisions = json.dumps([{"action": "tap", "coordinates": [100, 200]}])
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
|
||||
@@ -645,14 +565,6 @@ async def test_validator_pixel_validation_failure(mock_mcp, mock_context, temp_s
|
||||
"""Test that the pixel safety net fails and triggers FailureAnalyzer
|
||||
when Gemini reports target is missing.
|
||||
"""
|
||||
|
||||
def mock_call_tool(name, args):
|
||||
if name == "take_screenshot":
|
||||
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
|
||||
mock_mcp.call_tool.side_effect = mock_call_tool
|
||||
|
||||
decisions = json.dumps([{"action": "tap", "coordinates": [100, 200]}])
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
|
||||
@@ -697,23 +609,15 @@ async def test_validator_pixel_validation_failure(mock_mcp, mock_context, temp_s
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("artemis.agents.validator.validator.find_package")
|
||||
@patch("artemis.utils.app_launch_utils.launch_app_with_retries")
|
||||
async def test_validator_launch_app_passes_use_fallback_mcp(
|
||||
mock_launch_app, mock_find_package, mock_mcp, mock_context, temp_screenshot
|
||||
async def test_validator_launch_app_routes_through_manage_app(
|
||||
mock_mcp, mock_context, temp_screenshot
|
||||
):
|
||||
"""Test that ValidatorNode calls find_package with use_fallback=False
|
||||
for launch_app action in MCP mode.
|
||||
"""Test that a launch_app action item routes to the canonical manage_app tool.
|
||||
|
||||
Package resolution (find_package with use_fallback=False) now happens inside the
|
||||
actuator's manage_app implementation and is covered by the failure-analyzer tool
|
||||
tests; the Validator's contract is the translation and single dispatch.
|
||||
"""
|
||||
mock_find_package.return_value = "com.example.app"
|
||||
mock_launch_app.return_value = (True, None)
|
||||
|
||||
# Mock MCP calls: screenshot (pre), screenshot (post)
|
||||
mock_mcp.call_tool.side_effect = [
|
||||
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # pre-screenshot
|
||||
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # post-screenshot
|
||||
]
|
||||
|
||||
decisions = json.dumps([{"action": "launch_app", "app_name": "My App"}])
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
|
||||
@@ -721,27 +625,17 @@ async def test_validator_launch_app_passes_use_fallback_mcp(
|
||||
node = ValidatorNode(mock_context)
|
||||
result = await node(state)
|
||||
|
||||
mock_find_package.assert_called_once_with(mock_context, "My App", use_fallback=False)
|
||||
mock_launch_app.assert_called_once_with(mock_context, "com.example.app")
|
||||
manage_calls = mock_mcp.calls_for("manage_app")
|
||||
assert manage_calls == [{"action": "launch", "app_name": "My App"}]
|
||||
assert "last_execution_result" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("artemis.agents.validator.validator.find_package")
|
||||
@patch("artemis.utils.app_launch_utils.launch_app_with_retries")
|
||||
async def test_validator_launch_app_failure_no_retry(
|
||||
mock_launch_app, mock_find_package, mock_mcp, mock_context, temp_screenshot
|
||||
):
|
||||
async def test_validator_launch_app_failure_no_retry(mock_mcp, mock_context, temp_screenshot):
|
||||
"""Test that ValidatorNode does NOT retry launch_app action on failure."""
|
||||
mock_find_package.return_value = "com.example.app"
|
||||
mock_launch_app.return_value = (False, "Error: Force close")
|
||||
|
||||
def mock_call_tool(name, args):
|
||||
if name == "take_screenshot":
|
||||
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
|
||||
mock_mcp.call_tool.side_effect = mock_call_tool
|
||||
mock_mcp.action_handler = lambda name, args: (
|
||||
(False, "Error: Force close") if name == "manage_app" else (True, "")
|
||||
)
|
||||
|
||||
decisions = json.dumps([{"action": "launch_app", "app_name": "My App"}])
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
@@ -760,13 +654,9 @@ async def test_validator_launch_app_failure_no_retry(
|
||||
node = ValidatorNode(mock_context)
|
||||
result = await node(state)
|
||||
|
||||
# Verify find_package was called only once
|
||||
mock_find_package.assert_called_once_with(mock_context, "My App", use_fallback=False)
|
||||
# Verify launch_app_with_retries was called only once
|
||||
# (no retry by ValidatorNode itself since it relies on launch_app_with_retries)
|
||||
mock_launch_app.assert_called_once_with(mock_context, "com.example.app")
|
||||
# launch_app allows a single attempt -- no validator-level retry.
|
||||
assert len(mock_mcp.calls_for("manage_app")) == 1
|
||||
|
||||
# Verify attempts list has only 1 element
|
||||
assert "last_execution_result" in result
|
||||
report = result["last_execution_result"]
|
||||
assert len(report["execution"]) == 1
|
||||
@@ -783,7 +673,7 @@ async def test_validator_pre_execution_validation_disappeared_not_shifted(
|
||||
"""
|
||||
# Live hierarchy only contains a giant background/container
|
||||
# containing the text "Sign Up"
|
||||
mock_live_elements = [
|
||||
mock_mcp.hierarchy = [
|
||||
{
|
||||
"text": "Sign Up",
|
||||
"bounds": "[0,0][1080,2400]",
|
||||
@@ -791,15 +681,6 @@ async def test_validator_pre_execution_validation_disappeared_not_shifted(
|
||||
}
|
||||
]
|
||||
|
||||
def mock_call_tool(name, args):
|
||||
if name == "take_screenshot":
|
||||
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
|
||||
if name == "get_ui_hierarchy":
|
||||
return Mock(content=[Mock(text=json.dumps(mock_live_elements))])
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
|
||||
mock_mcp.call_tool.side_effect = mock_call_tool
|
||||
|
||||
decisions = json.dumps(
|
||||
[
|
||||
{
|
||||
@@ -850,12 +731,6 @@ async def test_validator_pre_execution_validation_ocr_direct_to_pixel(
|
||||
"""Test that OCR-derived elements bypass XML validation and
|
||||
directly use pixel-based validation.
|
||||
"""
|
||||
mock_mcp.call_tool.side_effect = [
|
||||
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # live screenshot for pixel check
|
||||
Mock(content=[Mock(text="Success")]), # tap action
|
||||
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # post-screenshot
|
||||
]
|
||||
|
||||
decisions = json.dumps(
|
||||
[
|
||||
{
|
||||
@@ -905,7 +780,7 @@ async def test_validator_pre_execution_xml_failure_not_overridden_when_pixel_byp
|
||||
"""Test that when XML validation fails AND Pixel validation is bypassed
|
||||
(PIXEL_BYPASSED), the XML failure is NOT overridden.
|
||||
"""
|
||||
mock_live_elements = [
|
||||
mock_mcp.hierarchy = [
|
||||
{
|
||||
"text": "Sign Up",
|
||||
"bounds": "[100,200][300,300]",
|
||||
@@ -913,15 +788,6 @@ async def test_validator_pre_execution_xml_failure_not_overridden_when_pixel_byp
|
||||
}
|
||||
]
|
||||
|
||||
def mock_call_tool(name, args):
|
||||
if name == "take_screenshot":
|
||||
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
|
||||
if name == "get_ui_hierarchy":
|
||||
return Mock(content=[Mock(text=json.dumps(mock_live_elements))])
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
|
||||
mock_mcp.call_tool.side_effect = mock_call_tool
|
||||
|
||||
decisions = json.dumps(
|
||||
[
|
||||
{
|
||||
@@ -970,17 +836,13 @@ async def test_validator_pre_execution_xml_failure_not_overridden_when_pixel_byp
|
||||
async def test_validator_failure_screenshot_mcp_error_handled_safely(
|
||||
mock_mcp, mock_context, temp_screenshot
|
||||
):
|
||||
"""Test that when take_screenshot fails on MCP server and returns an error text,
|
||||
the ValidatorNode does not crash with base64 decoding error.
|
||||
"""Test that when take_screenshot fails on the session, the ValidatorNode does
|
||||
not crash -- the failure screenshot is simply absent.
|
||||
"""
|
||||
def mock_call_tool(name, args):
|
||||
if name == "take_screenshot":
|
||||
return Mock(content=[Mock(text="Error: ADB connection lost")])
|
||||
if name == "tap":
|
||||
return Mock(content=[Mock(text="Error: Target not found")])
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
|
||||
mock_mcp.call_tool.side_effect = mock_call_tool
|
||||
mock_mcp.screenshot = RuntimeError("Error: ADB connection lost")
|
||||
mock_mcp.action_handler = lambda name, args: (
|
||||
(False, "Error: Target not found") if name == "click" else (True, "")
|
||||
)
|
||||
|
||||
decisions = json.dumps([{"action": "tap", "coordinates": [105, 205]}])
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
@@ -1002,4 +864,3 @@ async def test_validator_failure_screenshot_mcp_error_handled_safely(
|
||||
assert report["status"] == "failed"
|
||||
assert len(report["execution"]) == 1
|
||||
assert report["execution"][0]["action"] == "tap"
|
||||
|
||||
Reference in new issue
Block a user