feat: extract in-process MCP action layer, concurrent task dispatch, and spawn/reconcile hardening

- Extract device action execution into artemis/mcp action layer:
  typed ActionResult, capability manifest with startup validation,
  AdbActuator as the single physical implementation, in-memory
  ActionSession (replaces stdio subprocess MCP in validator), and
  McpActionExecutor preserving the legacy executor contract.
  adb_server keeps its external stdio/SSE surface unchanged and now
  reuses the actuator implementation; import-time side effects moved
  into configure_stdio_mode().
- Capability-gated prompt assembly (prompt_assembly.py): tool enums and
  teaching segments render per available backend tools; full-set output
  is byte-identical to the previous static text (locked by tests).
- Admin console task queue: serial worker replaced by a per-device
  concurrent dispatcher (ARTEMIS_CONCURRENCY_MODE=global restores
  serial semantics), per-session cancellation, strict device binding on
  enqueue, unified stdio piping with tee to trace stdout.log, and
  shutdown cleanup of all active runs. Fix _pid_exists exception order
  so PermissionError is treated as alive.
- mcp_server reliability: spawn watchdog kills runners stuck before
  interpreter startup and releases device reservations; task state
  reconcile uses psutil liveness, prefers runner-reported pids, and
  adds a 45s startup grace to stop false failure reports on Windows;
  conversation_id is now optional end-to-end; explicit device_serial is
  validated at submission.
- CLI install now writes ~/.claude/rules/artemis.md for Claude Code and
  migrates/removes the legacy CLAUDE.md injection block; ~/.claude.json
  is never rewritten when it fails to parse.
- Tests: Windows-portable stdio lifecycle and path assertions; new
  suites for action layer, prompt assembly, spawn watchdog, and task
  reconcile (895 passed, 3 skipped).
This commit is contained in:
weilinliu55 committed 2026-08-27 18:25:27 -07:00
1 parent 60312abbb0
commit cbdca1497c
52 files changed
+5735 -1649

No files matched your search

@@ -304,9 +304,11 @@ async def test_input_text_happy_path(mock_context, mock_controller):
ctx=mock_context,
)
assert "Executed typing 'hello'." in outcome
mock_controller.tap_at.assert_called_once_with(540, 1200)
# Focus goes through ensure_focus_at_coords (keyword-style tap), and typing
# never re-clears: the clear already happened via erase_text.
mock_controller.tap_at.assert_called_once_with(x=540, y=1200)
mock_controller.erase_text.assert_called_once()
mock_controller.type_text.assert_called_once_with("hello")
mock_controller.type_text.assert_called_once_with("hello", clear_existing=False)
@pytest.mark.asyncio
@@ -396,8 +398,10 @@ async def test_press_key_input_validation(mock_context, mock_controller):
controller=mock_controller,
ctx=mock_context,
)
# The current execution doesn't raise error on unknown keys, just logs error
assert "Error executing key press 'INVALID_KEY'." in outcome
# Unknown keycodes are forwarded verbatim to the driver (historical adb_server
# behavior); the mock driver accepts them, so the press reports success.
assert "Executed key press 'INVALID_KEY'." in outcome
mock_controller.press_key.assert_called_once_with("INVALID_KEY")
# ==========================================
@@ -409,8 +413,10 @@ async def test_press_key_input_validation(mock_context, mock_controller):
async def test_manage_app_happy_path(mock_context, mock_controller):
state = DummyState()
with (
patch("artemis.agents.validator.tool_declarations.find_package") as mock_find_package,
patch("artemis.agents.validator.tool_declarations.launch_app_with_retries") as mock_launch,
# The device-call bodies moved into the actuator layer, which imports these
# lazily from their source modules -- patch them at the source.
patch("artemis.tools.mobile.launch_app.find_package") as mock_find_package,
patch("artemis.utils.app_launch_utils.launch_app_with_retries") as mock_launch,
):
mock_find_package.return_value = "com.google.android.youtube"
mock_launch.return_value = (True, "")
@@ -430,7 +436,7 @@ async def test_manage_app_happy_path(mock_context, mock_controller):
@pytest.mark.asyncio
async def test_manage_app_input_validation(mock_context, mock_controller):
state = DummyState()
with patch("artemis.agents.validator.tool_declarations.find_package") as mock_find_package:
with patch("artemis.tools.mobile.launch_app.find_package") as mock_find_package:
mock_find_package.return_value = None
outcome, _, _, _ = await _exec_manage_app(
+233
View File
@@ -0,0 +1,233 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Prompt assembly tests: full-set output is byte-stable, absent tools leave no trace."""
from pathlib import Path
import re
from jinja2 import Template
import pytest
from artemis.agents.operator.prompt_builder import load_operator_prompts
from artemis.agents.operator.prompts import (
OPERATOR_PROMPT_TOOLSET,
apply_operator_prompt_contract,
)
from artemis.agents.prompt_assembly import gate_segment, render_tool_enum
from artemis.agents.validator.failure_analyzer import (
FailureAnalysisStrategy,
FailureAnalyzer,
PixelTargetDisappearedStrategy,
TargetDisappearedStrategy,
)
from artemis.agents.validator.validator import ValidationErrorCategory
REPO_ROOT = Path(__file__).resolve().parents[3]
OPERATOR_DEVICE_TOOLS = (
"click",
"input_text",
"swipe",
"press_key",
"manage_app",
"wait_for_delay",
"long_press",
)
def _tool_refs(text: str, tool: str) -> list[str]:
"""Finds real tool references (backticked or call form), ignoring prose."""
return re.findall(rf"(`{tool}`|\b{tool}\()", text)
# --- render_tool_enum ----------------------------------------------------------------
def test_enum_plain_list():
out = render_tool_enum(("a", "b", "c"), {"a", "b", "c"})
assert out == "`a`, `b`, `c`"
def test_enum_oxford_or_and():
tools = ("a", "b", "c")
assert render_tool_enum(tools, set(tools), final_sep="or") == "`a`, `b`, or `c`"
assert render_tool_enum(tools, set(tools), final_sep="and") == "`a`, `b`, and `c`"
def test_enum_two_items_no_oxford_comma():
assert render_tool_enum(("a", "b"), {"a", "b"}, final_sep="or") == "`a` or `b`"
def test_enum_single_and_empty():
assert render_tool_enum(("a", "b"), {"a"}) == "`a`"
assert render_tool_enum(("a", "b"), set()) == ""
def test_gate_segment():
assert gate_segment("teach", {"swipe"}, "swipe") == "teach"
assert gate_segment("teach", set(), "swipe") == ""
assert gate_segment("teach", {"a"}, "a", "b") == ""
# --- Operator prompt assembly --------------------------------------------------------
def test_full_toolset_is_the_default():
"""``available_tools=None`` must equal the explicit full set, byte for byte."""
for template in load_operator_prompts().values():
assert apply_operator_prompt_contract(template) == apply_operator_prompt_contract(
template, available_tools=OPERATOR_PROMPT_TOOLSET
)
@pytest.mark.parametrize("tool", OPERATOR_DEVICE_TOOLS + ("video_analyzer",))
def test_removed_tool_leaves_no_reference(tool):
"""The executable definition of 'an absent tool costs the model nothing'."""
for name, template in load_operator_prompts().items():
out = apply_operator_prompt_contract(
template, available_tools=OPERATOR_PROMPT_TOOLSET - {tool}
)
assert not _tool_refs(out, tool), f"'{tool}' still referenced in {name}"
def test_removed_swipe_takes_its_teaching_along():
"""Gating removes the drag/slider teaching, not just the backticked name."""
template = load_operator_prompts()["main_template"]
out = apply_operator_prompt_contract(
template, available_tools=OPERATOR_PROMPT_TOOLSET - {"swipe"}
)
assert "swipe/drag" not in out
assert "sliders/SeekBars" not in out
# The generic gesture-scope sentences survive.
assert "Action Scope & Interaction Precision" in out
def test_removed_wait_for_delay_keeps_interval_line_coherent():
"""Only the wait_for_delay parenthetical goes; the SOP grammar line stays."""
template = load_operator_prompts()["main_template"]
out = apply_operator_prompt_contract(
template, available_tools=OPERATOR_PROMPT_TOOLSET - {"wait_for_delay"}
)
assert "`Interval: <Interval>`: Declared polling cadence." in out
assert "Loading & Transitions" not in out
def test_reduced_enums_stay_well_formed():
template = load_operator_prompts()["main_template"]
out = apply_operator_prompt_contract(
template,
available_tools=OPERATOR_PROMPT_TOOLSET - {"manage_app", "wait_for_delay"},
)
assert (
"Physical device actions (`click`, `input_text`, `swipe`, `press_key`,"
" `long_press`)" in out
)
assert (
"Turn-Ending Action (`click`, `swipe`, `input_text`, `long_press`, or"
" `press_key`)" in out
)
def test_full_set_enum_slots_render_verbatim():
"""With every tool present the historical enumeration wording is reproduced."""
template = load_operator_prompts()["main_template"]
out = apply_operator_prompt_contract(template)
assert (
"Physical device actions (`click`, `input_text`, `swipe`, `press_key`,"
" `manage_app`, `wait_for_delay`, `long_press`)" in out
)
assert (
"Turn-Ending Action (`click`, `swipe`, `input_text`, `long_press`,"
" `press_key`, `manage_app`, or `wait_for_delay`)" in out
)
assert (
"(`ask_explorer`, `ask_diagnoser`, `video_analyzer`, `run_adb_command`,"
" `manage_task`, `analyze_task_output`, `read_note`, `list_notes`, and"
" `save_note`)" in out
)
# --- Flash prompt assembly -----------------------------------------------------------
def _render_flash(available_tools: frozenset[str]) -> str:
template_text = (REPO_ROOT / "artemis/agents/flash/flash_runner.md").read_text(
encoding="utf-8"
)
return Template(template_text).render(goal="test goal", available_tools=available_tools)
FLASH_FULL = frozenset({"manage_app", "wait_for_delay", "click_sequence", "click", "swipe"})
def test_flash_full_set_keeps_both_segments():
out = _render_flash(FLASH_FULL)
assert "**App Launching**" in out
assert "**Timed Waiting & Transitions**" in out
assert "{%" not in out
def test_flash_without_manage_app_drops_app_launching():
out = _render_flash(FLASH_FULL - {"manage_app"})
assert "**App Launching**" not in out
assert not _tool_refs(out, "manage_app")
# The neighbouring numbered rule survives.
assert "**Screenshot Pruning Awareness**" in out
def test_flash_without_wait_for_delay_drops_timed_waiting():
out = _render_flash(FLASH_FULL - {"wait_for_delay"})
assert "**Timed Waiting & Transitions**" not in out
assert not _tool_refs(out, "wait_for_delay")
assert "**State Drift & Progression Tolerance**" in out
# --- FailureAnalyzer strategy gating -------------------------------------------------
def _analyzer() -> FailureAnalyzer:
return FailureAnalyzer.__new__(FailureAnalyzer) # _select_strategy needs no ctx
def test_strategy_selection_unchanged_without_gating():
analyzer = _analyzer()
assert isinstance(
analyzer._select_strategy(ValidationErrorCategory.TARGET_DISAPPEARED),
TargetDisappearedStrategy,
)
assert isinstance(
analyzer._select_strategy(ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED),
PixelTargetDisappearedStrategy,
)
def test_strategy_falls_back_when_required_tool_missing():
"""Strategy + prompt are one unit: no click_sequence, no disappeared-prompts."""
analyzer = _analyzer()
strategy = analyzer._select_strategy(
ValidationErrorCategory.TARGET_DISAPPEARED,
available_tools=frozenset({"click", "swipe"}),
)
assert type(strategy) is FailureAnalysisStrategy
assert strategy.get_prompt_template_name() == "failure_analyzer.md"
def test_strategy_kept_when_required_tool_present():
analyzer = _analyzer()
strategy = analyzer._select_strategy(
ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED,
available_tools=frozenset({"click_sequence"}),
)
assert isinstance(strategy, PixelTargetDisappearedStrategy)
+112 -251
View File
@@ -12,13 +12,25 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""ValidatorNode tests over the in-process unified action session.
The Validator no longer spawns a stdio MCP subprocess; it calls a shared
``ActionSession`` whose device actions return structured ``ActionResult`` objects.
``FakeActionSession`` below stands in for it: ``action_handler`` decides each
action's ``(ok, message)``, ``screenshot`` feeds the polling loop, and ``hierarchy``
feeds the safety-net XML validation.
"""
import json
from unittest.mock import AsyncMock, MagicMock, Mock, patch
from langchain_core.messages import AIMessage
import pytest
from artemis.agents.validator.failure_analyzer import ValidationErrorCategory
from artemis.agents.validator.validator import ValidatorNode
from artemis.context import ArtemisContext
import pytest
from artemis.mcp.action_types import ActionCode, ActionResult
class DummyState:
@@ -38,6 +50,46 @@ class DummyState:
return update
class FakeActionSession:
"""Test double for artemis.mcp.action_session.ActionSession."""
def __init__(self):
self.started = True
self.calls: list[tuple[str, dict]] = []
# (name, args) -> (ok, message)
self.action_handler = lambda name, args: (True, "")
# str | callable | Exception
self.screenshot = "ZHVtbXlfZGF0YQ==" # b64("dummy_data")
self.hierarchy: list | Exception = []
async def call(self, name, args):
self.calls.append((name, args))
ok, msg = self.action_handler(name, args)
return ActionResult(
ok=ok,
code=ActionCode.OK if ok else ActionCode.DEVICE_ERROR,
action=name,
message=msg,
)
async def screenshot_b64(self, timeout=None):
shot = self.screenshot
if isinstance(shot, Exception):
raise shot
return shot() if callable(shot) else shot
async def ui_hierarchy(self, timeout=None):
if isinstance(self.hierarchy, Exception):
raise self.hierarchy
return self.hierarchy
async def aclose(self):
pass
def calls_for(self, name: str) -> list[dict]:
return [args for n, args in self.calls if n == name]
@pytest.fixture
def mock_context(tmp_path):
ctx = Mock(spec=ArtemisContext)
@@ -45,6 +97,9 @@ def mock_context(tmp_path):
ctx.data_engine = Mock()
ctx.data_engine.base_dir = tmp_path
ctx.data_engine.get_relative_time.return_value = 1.0
ctx.device = Mock()
ctx.device.device_width = 1080
ctx.device.device_height = 2400
return ctx
@@ -57,31 +112,18 @@ def temp_screenshot(tmp_path):
@pytest.fixture
def mock_mcp():
with (
patch("mcp.client.stdio.stdio_client") as mock_stdio,
patch("artemis.agents.validator.validator.ClientSession") as mock_session_cls,
):
mock_ctx = AsyncMock()
mock_ctx.__aenter__.return_value = (Mock(), Mock())
mock_stdio.return_value = mock_ctx
fake = FakeActionSession()
mock_session = AsyncMock()
mock_session.initialize = AsyncMock()
mock_session_cls.return_value = mock_session
async def _get_session(ctx, actuator=None):
return fake
yield mock_session
with patch("artemis.agents.validator.validator.get_action_session", _get_session):
yield fake
@pytest.mark.asyncio
async def test_validator_success(mock_mcp, mock_context, temp_screenshot):
"""Test that ValidatorNode executes actions successfully."""
# Mock MCP calls: screenshot, tap, screenshot
mock_mcp.call_tool.side_effect = [
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # pre-screenshot
Mock(content=[Mock(text="Success")]), # tap action
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # post-screenshot
]
decisions = json.dumps([{"action": "tap", "coordinates": [105, 205]}])
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
@@ -95,78 +137,23 @@ async def test_validator_success(mock_mcp, mock_context, temp_screenshot):
assert len(report["execution"]) == 1
assert report["execution"][0]["action"] == "tap"
assert "attempts" not in report["execution"][0]
# The Operator's tap verb was translated to the canonical click tool.
assert len(mock_mcp.calls_for("click")) == 1
@pytest.mark.asyncio
async def test_failed_mcp_handshake_exits_contexts_and_disables_child_awake_policy():
"""A handshake failure must not leave AnyIO cancellation scopes behind."""
events = []
class FakeClientContext:
async def __aenter__(self):
events.append("client_enter")
return Mock(), Mock()
async def __aexit__(self, exc_type, exc, traceback):
events.append("client_exit")
class FakeSession:
async def __aenter__(self):
events.append("session_enter")
return self
async def __aexit__(self, exc_type, exc, traceback):
events.append("session_exit")
async def initialize(self):
raise TimeoutError("handshake timed out")
ctx = Mock(spec=ArtemisContext)
ctx.mcp_client_ctx = None
ctx.mcp_session = None
fake_client_ctx = FakeClientContext()
fake_session = FakeSession()
captured_parameters = []
def fake_stdio_client(parameters):
captured_parameters.append(parameters)
return fake_client_ctx
node = ValidatorNode(ctx)
with (
patch("artemis.agents.validator.validator.stdio_client", fake_stdio_client),
patch("artemis.agents.validator.validator.ClientSession", return_value=fake_session),
patch("artemis.agents.validator.validator.AdbClient") as adb_client,
patch.object(node, "_kill_mcp_server_instantly") as emergency_kill,
):
adb_client.return_value.device_list.return_value = []
result = await node._get_mcp_session()
assert result is None
assert events == ["client_enter", "session_enter", "session_exit", "client_exit"]
assert ctx.mcp_session is None
assert ctx.mcp_client_ctx is None
assert captured_parameters[0].env["ARTEMIS_KEEP_DEVICE_AWAKE"] == "false"
emergency_kill.assert_not_called()
# NOTE: upstream's test_failed_mcp_handshake_exits_contexts_and_disables_child_awake_policy
# guarded the stdio-subprocess handshake teardown, machinery this migration removed
# entirely. Its hazard class (leaked AnyIO cancel scopes) is now pinned by
# tests/unit/mcp/test_action_session.py against the in-process ActionSession.
@pytest.mark.asyncio
async def test_validator_failure_analysis(mock_mcp, mock_context, temp_screenshot):
"""Test that ValidatorNode triggers failure analysis on error and handles
cannot_fix via tool.
"""
tap_count = 0
def mock_call_tool(name, args):
nonlocal tap_count
if name == "take_screenshot":
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
if name == "tap":
tap_count += 1
return Mock(content=[Mock(text="Error: Element not found")])
return Mock(content=[Mock(text="Success")])
mock_mcp.call_tool.side_effect = mock_call_tool
mock_mcp.action_handler = lambda name, args: (
(False, "Error: Element not found") if name == "click" else (True, "")
)
decisions = json.dumps([{"action": "tap", "coordinates": [105, 205]}])
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
@@ -203,18 +190,15 @@ async def test_validator_repair_success(mock_mcp, mock_context, temp_screenshot)
"""
tap_count = 0
def mock_call_tool(name, args):
def handler(name, args):
nonlocal tap_count
if name == "take_screenshot":
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
if name == "tap":
if name == "click":
tap_count += 1
if tap_count <= 2: # First tap fails
return Mock(content=[Mock(text="Error: Element not found")])
return Mock(content=[Mock(text="Success")])
return Mock(content=[Mock(text="Success")])
if tap_count <= 2: # First action fails on both validator attempts
return False, "Error: Element not found"
return True, ""
mock_mcp.call_tool.side_effect = mock_call_tool
mock_mcp.action_handler = handler
# Two actions planned initially
decisions = json.dumps(
@@ -254,12 +238,6 @@ async def test_validator_repair_success(mock_mcp, mock_context, temp_screenshot)
@pytest.mark.asyncio
async def test_validator_wait_for_delay(mock_mcp, mock_context, temp_screenshot):
"""Test that wait_for_delay executes successfully."""
def mock_call(name, args):
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
mock_mcp.call_tool.side_effect = mock_call
decisions = json.dumps([{"action": "wait_for_delay", "time_in_ms": 10}])
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
@@ -272,19 +250,13 @@ async def test_validator_wait_for_delay(mock_mcp, mock_context, temp_screenshot)
assert "execution" in report
assert len(report["execution"]) == 1
assert report["execution"][0]["action"] == "wait_for_delay"
# wait_for_delay is a pure client-side sleep, never a session call.
assert mock_mcp.calls_for("wait_for_delay") == []
@pytest.mark.asyncio
async def test_validator_focus_and_clear_text_no_ui_change(mock_mcp, mock_context, temp_screenshot):
"""Test that focus_and_clear_text executes successfully even if no UI change is detected."""
def mock_call(name, args):
if name == "focus_and_clear_text":
return Mock(content=[Mock(text="Success")])
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
mock_mcp.call_tool.side_effect = mock_call
decisions = json.dumps([{"action": "focus_and_clear_text", "coordinates": [400, 500]}])
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
@@ -297,6 +269,7 @@ async def test_validator_focus_and_clear_text_no_ui_change(mock_mcp, mock_contex
assert "execution" in report
assert len(report["execution"]) == 1
assert report["execution"][0]["action"] == "focus_and_clear_text"
assert len(mock_mcp.calls_for("focus_and_clear_text")) == 1
@pytest.mark.asyncio
@@ -304,16 +277,6 @@ async def test_validator_silent_failure_treated_as_success(mock_mcp, mock_contex
"""Test that silent failure (exec succeeds but no UI change) is treated as
success and does not trigger FailureAnalyzer.
"""
def mock_call_tool(name, args):
if name == "take_screenshot":
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
if name == "tap":
return Mock(content=[Mock(text="Success")])
return Mock(content=[Mock(text="Success")])
mock_mcp.call_tool.side_effect = mock_call_tool
decisions = json.dumps([{"action": "tap", "coordinates": [105, 205]}])
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
@@ -344,13 +307,7 @@ async def test_validator_records_skipped_actions_on_cannot_fix(
mock_mcp, mock_context, temp_screenshot
):
"""Test that ValidatorNode marks unexecuted actions as Skipped when repair fails."""
def mock_call_tool(name, args):
if name == "take_screenshot":
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
return Mock(content=[Mock(text="Error: Connection lost")])
mock_mcp.call_tool.side_effect = mock_call_tool
mock_mcp.action_handler = lambda name, args: (False, "Error: Connection lost")
# Two actions planned
decisions = json.dumps(
@@ -399,7 +356,7 @@ async def test_validator_pre_execution_validation_and_repair(
"""Test that pre-execution validation fails when element changes, and is
repaired by FailureAnalyzer.
"""
mock_live_elements = [
mock_mcp.hierarchy = [
{
"text": "Sign Up",
"bounds": "[100,200][300,300]",
@@ -407,17 +364,6 @@ async def test_validator_pre_execution_validation_and_repair(
}
]
def mock_call_tool(name, args):
if name == "take_screenshot":
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
if name == "get_ui_hierarchy":
return Mock(content=[Mock(text=json.dumps(mock_live_elements))])
if name == "tap":
return Mock(content=[Mock(text="Success")])
return Mock(content=[Mock(text="Success")])
mock_mcp.call_tool.side_effect = mock_call_tool
decisions = json.dumps(
[
{
@@ -477,7 +423,7 @@ async def test_validator_pre_execution_validation_self_healing(
mock_mcp, mock_context, temp_screenshot
):
"""Test that pre-execution validation succeeds and self-heals when element shifts slightly."""
mock_live_elements = [
mock_mcp.hierarchy = [
{
"text": "Login",
"bounds": "[100,220][300,320]",
@@ -485,21 +431,6 @@ async def test_validator_pre_execution_validation_self_healing(
}
]
tapped_coordinates = None
def mock_call_tool(name, args):
nonlocal tapped_coordinates
if name == "take_screenshot":
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
if name == "get_ui_hierarchy":
return Mock(content=[Mock(text=json.dumps(mock_live_elements))])
if name == "tap":
tapped_coordinates = args.get("coordinates")
return Mock(content=[Mock(text="Success")])
return Mock(content=[Mock(text="Success")])
mock_mcp.call_tool.side_effect = mock_call_tool
decisions = json.dumps(
[
{
@@ -518,8 +449,12 @@ async def test_validator_pre_execution_validation_self_healing(
with patch("artemis.utils.image_diff.check_ui_change", return_value=True):
await node(state)
# Center of [100, 220][300, 320] is [200, 270]
assert tapped_coordinates == [200, 270]
# Center of [100, 220][300, 320] is pixel [200, 270]; the healed coordinates
# travel to the canonical click tool normalized to 0-1000 on a 1080x2400 screen:
# round(200 * 1000 / 1080) = 185, round(270 * 1000 / 2400) = 112.
clicks = mock_mcp.calls_for("click")
assert len(clicks) == 1
assert clicks[0]["target"] == [185, 112]
@pytest.mark.asyncio
@@ -531,7 +466,7 @@ async def test_validator_pre_execution_validation_anonymous_occupant(
"""
# The live screen contains an anonymous clickable view covering
# target coordinates [200, 250]
mock_live_elements = [
mock_mcp.hierarchy = [
{
"text": "",
"bounds": "[150,200][250,300]",
@@ -540,15 +475,6 @@ async def test_validator_pre_execution_validation_anonymous_occupant(
}
]
def mock_call_tool(name, args):
if name == "take_screenshot":
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
if name == "get_ui_hierarchy":
return Mock(content=[Mock(text=json.dumps(mock_live_elements))])
return Mock(content=[Mock(text="Success")])
mock_mcp.call_tool.side_effect = mock_call_tool
decisions = json.dumps(
[
{
@@ -596,12 +522,6 @@ async def test_validator_pre_execution_validation_anonymous_occupant(
@pytest.mark.asyncio
async def test_validator_pixel_validation_success(mock_mcp, mock_context, temp_screenshot):
"""Test that the pixel safety net succeeds when Gemini reports target is present."""
mock_mcp.call_tool.side_effect = [
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # live screenshot for pixel check
Mock(content=[Mock(text="Success")]), # tap action
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # post-screenshot
]
decisions = json.dumps([{"action": "tap", "coordinates": [100, 200]}])
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
@@ -645,14 +565,6 @@ async def test_validator_pixel_validation_failure(mock_mcp, mock_context, temp_s
"""Test that the pixel safety net fails and triggers FailureAnalyzer
when Gemini reports target is missing.
"""
def mock_call_tool(name, args):
if name == "take_screenshot":
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
return Mock(content=[Mock(text="Success")])
mock_mcp.call_tool.side_effect = mock_call_tool
decisions = json.dumps([{"action": "tap", "coordinates": [100, 200]}])
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
@@ -697,23 +609,15 @@ async def test_validator_pixel_validation_failure(mock_mcp, mock_context, temp_s
@pytest.mark.asyncio
@patch("artemis.agents.validator.validator.find_package")
@patch("artemis.utils.app_launch_utils.launch_app_with_retries")
async def test_validator_launch_app_passes_use_fallback_mcp(
mock_launch_app, mock_find_package, mock_mcp, mock_context, temp_screenshot
async def test_validator_launch_app_routes_through_manage_app(
mock_mcp, mock_context, temp_screenshot
):
"""Test that ValidatorNode calls find_package with use_fallback=False
for launch_app action in MCP mode.
"""Test that a launch_app action item routes to the canonical manage_app tool.
Package resolution (find_package with use_fallback=False) now happens inside the
actuator's manage_app implementation and is covered by the failure-analyzer tool
tests; the Validator's contract is the translation and single dispatch.
"""
mock_find_package.return_value = "com.example.app"
mock_launch_app.return_value = (True, None)
# Mock MCP calls: screenshot (pre), screenshot (post)
mock_mcp.call_tool.side_effect = [
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # pre-screenshot
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # post-screenshot
]
decisions = json.dumps([{"action": "launch_app", "app_name": "My App"}])
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
@@ -721,27 +625,17 @@ async def test_validator_launch_app_passes_use_fallback_mcp(
node = ValidatorNode(mock_context)
result = await node(state)
mock_find_package.assert_called_once_with(mock_context, "My App", use_fallback=False)
mock_launch_app.assert_called_once_with(mock_context, "com.example.app")
manage_calls = mock_mcp.calls_for("manage_app")
assert manage_calls == [{"action": "launch", "app_name": "My App"}]
assert "last_execution_result" in result
@pytest.mark.asyncio
@patch("artemis.agents.validator.validator.find_package")
@patch("artemis.utils.app_launch_utils.launch_app_with_retries")
async def test_validator_launch_app_failure_no_retry(
mock_launch_app, mock_find_package, mock_mcp, mock_context, temp_screenshot
):
async def test_validator_launch_app_failure_no_retry(mock_mcp, mock_context, temp_screenshot):
"""Test that ValidatorNode does NOT retry launch_app action on failure."""
mock_find_package.return_value = "com.example.app"
mock_launch_app.return_value = (False, "Error: Force close")
def mock_call_tool(name, args):
if name == "take_screenshot":
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
return Mock(content=[Mock(text="Success")])
mock_mcp.call_tool.side_effect = mock_call_tool
mock_mcp.action_handler = lambda name, args: (
(False, "Error: Force close") if name == "manage_app" else (True, "")
)
decisions = json.dumps([{"action": "launch_app", "app_name": "My App"}])
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
@@ -760,13 +654,9 @@ async def test_validator_launch_app_failure_no_retry(
node = ValidatorNode(mock_context)
result = await node(state)
# Verify find_package was called only once
mock_find_package.assert_called_once_with(mock_context, "My App", use_fallback=False)
# Verify launch_app_with_retries was called only once
# (no retry by ValidatorNode itself since it relies on launch_app_with_retries)
mock_launch_app.assert_called_once_with(mock_context, "com.example.app")
# launch_app allows a single attempt -- no validator-level retry.
assert len(mock_mcp.calls_for("manage_app")) == 1
# Verify attempts list has only 1 element
assert "last_execution_result" in result
report = result["last_execution_result"]
assert len(report["execution"]) == 1
@@ -783,7 +673,7 @@ async def test_validator_pre_execution_validation_disappeared_not_shifted(
"""
# Live hierarchy only contains a giant background/container
# containing the text "Sign Up"
mock_live_elements = [
mock_mcp.hierarchy = [
{
"text": "Sign Up",
"bounds": "[0,0][1080,2400]",
@@ -791,15 +681,6 @@ async def test_validator_pre_execution_validation_disappeared_not_shifted(
}
]
def mock_call_tool(name, args):
if name == "take_screenshot":
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
if name == "get_ui_hierarchy":
return Mock(content=[Mock(text=json.dumps(mock_live_elements))])
return Mock(content=[Mock(text="Success")])
mock_mcp.call_tool.side_effect = mock_call_tool
decisions = json.dumps(
[
{
@@ -850,12 +731,6 @@ async def test_validator_pre_execution_validation_ocr_direct_to_pixel(
"""Test that OCR-derived elements bypass XML validation and
directly use pixel-based validation.
"""
mock_mcp.call_tool.side_effect = [
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # live screenshot for pixel check
Mock(content=[Mock(text="Success")]), # tap action
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # post-screenshot
]
decisions = json.dumps(
[
{
@@ -905,7 +780,7 @@ async def test_validator_pre_execution_xml_failure_not_overridden_when_pixel_byp
"""Test that when XML validation fails AND Pixel validation is bypassed
(PIXEL_BYPASSED), the XML failure is NOT overridden.
"""
mock_live_elements = [
mock_mcp.hierarchy = [
{
"text": "Sign Up",
"bounds": "[100,200][300,300]",
@@ -913,15 +788,6 @@ async def test_validator_pre_execution_xml_failure_not_overridden_when_pixel_byp
}
]
def mock_call_tool(name, args):
if name == "take_screenshot":
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
if name == "get_ui_hierarchy":
return Mock(content=[Mock(text=json.dumps(mock_live_elements))])
return Mock(content=[Mock(text="Success")])
mock_mcp.call_tool.side_effect = mock_call_tool
decisions = json.dumps(
[
{
@@ -970,17 +836,13 @@ async def test_validator_pre_execution_xml_failure_not_overridden_when_pixel_byp
async def test_validator_failure_screenshot_mcp_error_handled_safely(
mock_mcp, mock_context, temp_screenshot
):
"""Test that when take_screenshot fails on MCP server and returns an error text,
the ValidatorNode does not crash with base64 decoding error.
"""Test that when take_screenshot fails on the session, the ValidatorNode does
not crash -- the failure screenshot is simply absent.
"""
def mock_call_tool(name, args):
if name == "take_screenshot":
return Mock(content=[Mock(text="Error: ADB connection lost")])
if name == "tap":
return Mock(content=[Mock(text="Error: Target not found")])
return Mock(content=[Mock(text="Success")])
mock_mcp.call_tool.side_effect = mock_call_tool
mock_mcp.screenshot = RuntimeError("Error: ADB connection lost")
mock_mcp.action_handler = lambda name, args: (
(False, "Error: Target not found") if name == "click" else (True, "")
)
decisions = json.dumps([{"action": "tap", "coordinates": [105, 205]}])
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
@@ -1002,4 +864,3 @@ async def test_validator_failure_screenshot_mcp_error_handled_safely(
assert report["status"] == "failed"
assert len(report["execution"]) == 1
assert report["execution"][0]["action"] == "tap"