feat: extract in-process MCP action layer, concurrent task dispatch, and spawn/reconcile hardening
- Extract device action execution into artemis/mcp action layer: typed ActionResult, capability manifest with startup validation, AdbActuator as the single physical implementation, in-memory ActionSession (replaces stdio subprocess MCP in validator), and McpActionExecutor preserving the legacy executor contract. adb_server keeps its external stdio/SSE surface unchanged and now reuses the actuator implementation; import-time side effects moved into configure_stdio_mode(). - Capability-gated prompt assembly (prompt_assembly.py): tool enums and teaching segments render per available backend tools; full-set output is byte-identical to the previous static text (locked by tests). - Admin console task queue: serial worker replaced by a per-device concurrent dispatcher (ARTEMIS_CONCURRENCY_MODE=global restores serial semantics), per-session cancellation, strict device binding on enqueue, unified stdio piping with tee to trace stdout.log, and shutdown cleanup of all active runs. Fix _pid_exists exception order so PermissionError is treated as alive. - mcp_server reliability: spawn watchdog kills runners stuck before interpreter startup and releases device reservations; task state reconcile uses psutil liveness, prefers runner-reported pids, and adds a 45s startup grace to stop false failure reports on Windows; conversation_id is now optional end-to-end; explicit device_serial is validated at submission. - CLI install now writes ~/.claude/rules/artemis.md for Claude Code and migrates/removes the legacy CLAUDE.md injection block; ~/.claude.json is never rewritten when it fails to parse. - Tests: Windows-portable stdio lifecycle and path assertions; new suites for action layer, prompt assembly, spawn watchdog, and task reconcile (895 passed, 3 skipped).
This commit is contained in:
1 parent
60312abbb0
commit
cbdca1497c
52 files changed
+5735
-1649
No files matched your search
@@ -667,6 +667,10 @@ async def test_queue_worker_notifies_conversation():
|
||||
patch("apps.admin_console.services.task_queue_service.media_service"),
|
||||
patch("mcp_server.notifiers.notify") as mock_notify,
|
||||
patch("apps.admin_console.services.task_queue_service.session_repo") as mock_repo,
|
||||
patch(
|
||||
"artemis.runtime.device_pool.device_pool.list_devices_async",
|
||||
new=AsyncMock(return_value=[]),
|
||||
),
|
||||
):
|
||||
mock_repo.get_running_session_id.return_value = None
|
||||
|
||||
@@ -849,6 +853,11 @@ async def test_enqueue_tasks_debounces_rapid_identical_submissions():
|
||||
with (
|
||||
patch("apps.admin_console.services.task_queue_service.session_repo"),
|
||||
patch("apps.admin_console.services.task_queue_service.DeviceExecutionLock.reserve", return_value="ticket-456"),
|
||||
patch(
|
||||
"artemis.runtime.device_pool.device_pool.list_devices_async",
|
||||
new=AsyncMock(return_value=[]),
|
||||
),
|
||||
patch.object(TaskQueueService, "ensure_worker_running"),
|
||||
):
|
||||
res1 = await task_queue_service.enqueue_tasks(["Rapid duplicate goal"], device_serial="dev-1")
|
||||
assert len(state.queue_items) == 1
|
||||
|
||||
@@ -304,9 +304,11 @@ async def test_input_text_happy_path(mock_context, mock_controller):
|
||||
ctx=mock_context,
|
||||
)
|
||||
assert "Executed typing 'hello'." in outcome
|
||||
mock_controller.tap_at.assert_called_once_with(540, 1200)
|
||||
# Focus goes through ensure_focus_at_coords (keyword-style tap), and typing
|
||||
# never re-clears: the clear already happened via erase_text.
|
||||
mock_controller.tap_at.assert_called_once_with(x=540, y=1200)
|
||||
mock_controller.erase_text.assert_called_once()
|
||||
mock_controller.type_text.assert_called_once_with("hello")
|
||||
mock_controller.type_text.assert_called_once_with("hello", clear_existing=False)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -396,8 +398,10 @@ async def test_press_key_input_validation(mock_context, mock_controller):
|
||||
controller=mock_controller,
|
||||
ctx=mock_context,
|
||||
)
|
||||
# The current execution doesn't raise error on unknown keys, just logs error
|
||||
assert "Error executing key press 'INVALID_KEY'." in outcome
|
||||
# Unknown keycodes are forwarded verbatim to the driver (historical adb_server
|
||||
# behavior); the mock driver accepts them, so the press reports success.
|
||||
assert "Executed key press 'INVALID_KEY'." in outcome
|
||||
mock_controller.press_key.assert_called_once_with("INVALID_KEY")
|
||||
|
||||
|
||||
# ==========================================
|
||||
@@ -409,8 +413,10 @@ async def test_press_key_input_validation(mock_context, mock_controller):
|
||||
async def test_manage_app_happy_path(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
with (
|
||||
patch("artemis.agents.validator.tool_declarations.find_package") as mock_find_package,
|
||||
patch("artemis.agents.validator.tool_declarations.launch_app_with_retries") as mock_launch,
|
||||
# The device-call bodies moved into the actuator layer, which imports these
|
||||
# lazily from their source modules -- patch them at the source.
|
||||
patch("artemis.tools.mobile.launch_app.find_package") as mock_find_package,
|
||||
patch("artemis.utils.app_launch_utils.launch_app_with_retries") as mock_launch,
|
||||
):
|
||||
mock_find_package.return_value = "com.google.android.youtube"
|
||||
mock_launch.return_value = (True, "")
|
||||
@@ -430,7 +436,7 @@ async def test_manage_app_happy_path(mock_context, mock_controller):
|
||||
@pytest.mark.asyncio
|
||||
async def test_manage_app_input_validation(mock_context, mock_controller):
|
||||
state = DummyState()
|
||||
with patch("artemis.agents.validator.tool_declarations.find_package") as mock_find_package:
|
||||
with patch("artemis.tools.mobile.launch_app.find_package") as mock_find_package:
|
||||
mock_find_package.return_value = None
|
||||
|
||||
outcome, _, _, _ = await _exec_manage_app(
|
||||
|
||||
@@ -0,0 +1,233 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Prompt assembly tests: full-set output is byte-stable, absent tools leave no trace."""
|
||||
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
from jinja2 import Template
|
||||
import pytest
|
||||
|
||||
from artemis.agents.operator.prompt_builder import load_operator_prompts
|
||||
from artemis.agents.operator.prompts import (
|
||||
OPERATOR_PROMPT_TOOLSET,
|
||||
apply_operator_prompt_contract,
|
||||
)
|
||||
from artemis.agents.prompt_assembly import gate_segment, render_tool_enum
|
||||
from artemis.agents.validator.failure_analyzer import (
|
||||
FailureAnalysisStrategy,
|
||||
FailureAnalyzer,
|
||||
PixelTargetDisappearedStrategy,
|
||||
TargetDisappearedStrategy,
|
||||
)
|
||||
from artemis.agents.validator.validator import ValidationErrorCategory
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
|
||||
OPERATOR_DEVICE_TOOLS = (
|
||||
"click",
|
||||
"input_text",
|
||||
"swipe",
|
||||
"press_key",
|
||||
"manage_app",
|
||||
"wait_for_delay",
|
||||
"long_press",
|
||||
)
|
||||
|
||||
|
||||
def _tool_refs(text: str, tool: str) -> list[str]:
|
||||
"""Finds real tool references (backticked or call form), ignoring prose."""
|
||||
return re.findall(rf"(`{tool}`|\b{tool}\()", text)
|
||||
|
||||
|
||||
# --- render_tool_enum ----------------------------------------------------------------
|
||||
|
||||
|
||||
def test_enum_plain_list():
|
||||
out = render_tool_enum(("a", "b", "c"), {"a", "b", "c"})
|
||||
assert out == "`a`, `b`, `c`"
|
||||
|
||||
|
||||
def test_enum_oxford_or_and():
|
||||
tools = ("a", "b", "c")
|
||||
assert render_tool_enum(tools, set(tools), final_sep="or") == "`a`, `b`, or `c`"
|
||||
assert render_tool_enum(tools, set(tools), final_sep="and") == "`a`, `b`, and `c`"
|
||||
|
||||
|
||||
def test_enum_two_items_no_oxford_comma():
|
||||
assert render_tool_enum(("a", "b"), {"a", "b"}, final_sep="or") == "`a` or `b`"
|
||||
|
||||
|
||||
def test_enum_single_and_empty():
|
||||
assert render_tool_enum(("a", "b"), {"a"}) == "`a`"
|
||||
assert render_tool_enum(("a", "b"), set()) == ""
|
||||
|
||||
|
||||
def test_gate_segment():
|
||||
assert gate_segment("teach", {"swipe"}, "swipe") == "teach"
|
||||
assert gate_segment("teach", set(), "swipe") == ""
|
||||
assert gate_segment("teach", {"a"}, "a", "b") == ""
|
||||
|
||||
|
||||
# --- Operator prompt assembly --------------------------------------------------------
|
||||
|
||||
|
||||
def test_full_toolset_is_the_default():
|
||||
"""``available_tools=None`` must equal the explicit full set, byte for byte."""
|
||||
for template in load_operator_prompts().values():
|
||||
assert apply_operator_prompt_contract(template) == apply_operator_prompt_contract(
|
||||
template, available_tools=OPERATOR_PROMPT_TOOLSET
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tool", OPERATOR_DEVICE_TOOLS + ("video_analyzer",))
|
||||
def test_removed_tool_leaves_no_reference(tool):
|
||||
"""The executable definition of 'an absent tool costs the model nothing'."""
|
||||
for name, template in load_operator_prompts().items():
|
||||
out = apply_operator_prompt_contract(
|
||||
template, available_tools=OPERATOR_PROMPT_TOOLSET - {tool}
|
||||
)
|
||||
assert not _tool_refs(out, tool), f"'{tool}' still referenced in {name}"
|
||||
|
||||
|
||||
def test_removed_swipe_takes_its_teaching_along():
|
||||
"""Gating removes the drag/slider teaching, not just the backticked name."""
|
||||
template = load_operator_prompts()["main_template"]
|
||||
out = apply_operator_prompt_contract(
|
||||
template, available_tools=OPERATOR_PROMPT_TOOLSET - {"swipe"}
|
||||
)
|
||||
assert "swipe/drag" not in out
|
||||
assert "sliders/SeekBars" not in out
|
||||
# The generic gesture-scope sentences survive.
|
||||
assert "Action Scope & Interaction Precision" in out
|
||||
|
||||
|
||||
def test_removed_wait_for_delay_keeps_interval_line_coherent():
|
||||
"""Only the wait_for_delay parenthetical goes; the SOP grammar line stays."""
|
||||
template = load_operator_prompts()["main_template"]
|
||||
out = apply_operator_prompt_contract(
|
||||
template, available_tools=OPERATOR_PROMPT_TOOLSET - {"wait_for_delay"}
|
||||
)
|
||||
assert "`Interval: <Interval>`: Declared polling cadence." in out
|
||||
assert "Loading & Transitions" not in out
|
||||
|
||||
|
||||
def test_reduced_enums_stay_well_formed():
|
||||
template = load_operator_prompts()["main_template"]
|
||||
out = apply_operator_prompt_contract(
|
||||
template,
|
||||
available_tools=OPERATOR_PROMPT_TOOLSET - {"manage_app", "wait_for_delay"},
|
||||
)
|
||||
assert (
|
||||
"Physical device actions (`click`, `input_text`, `swipe`, `press_key`,"
|
||||
" `long_press`)" in out
|
||||
)
|
||||
assert (
|
||||
"Turn-Ending Action (`click`, `swipe`, `input_text`, `long_press`, or"
|
||||
" `press_key`)" in out
|
||||
)
|
||||
|
||||
|
||||
def test_full_set_enum_slots_render_verbatim():
|
||||
"""With every tool present the historical enumeration wording is reproduced."""
|
||||
template = load_operator_prompts()["main_template"]
|
||||
out = apply_operator_prompt_contract(template)
|
||||
assert (
|
||||
"Physical device actions (`click`, `input_text`, `swipe`, `press_key`,"
|
||||
" `manage_app`, `wait_for_delay`, `long_press`)" in out
|
||||
)
|
||||
assert (
|
||||
"Turn-Ending Action (`click`, `swipe`, `input_text`, `long_press`,"
|
||||
" `press_key`, `manage_app`, or `wait_for_delay`)" in out
|
||||
)
|
||||
assert (
|
||||
"(`ask_explorer`, `ask_diagnoser`, `video_analyzer`, `run_adb_command`,"
|
||||
" `manage_task`, `analyze_task_output`, `read_note`, `list_notes`, and"
|
||||
" `save_note`)" in out
|
||||
)
|
||||
|
||||
|
||||
# --- Flash prompt assembly -----------------------------------------------------------
|
||||
|
||||
|
||||
def _render_flash(available_tools: frozenset[str]) -> str:
|
||||
template_text = (REPO_ROOT / "artemis/agents/flash/flash_runner.md").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
return Template(template_text).render(goal="test goal", available_tools=available_tools)
|
||||
|
||||
|
||||
FLASH_FULL = frozenset({"manage_app", "wait_for_delay", "click_sequence", "click", "swipe"})
|
||||
|
||||
|
||||
def test_flash_full_set_keeps_both_segments():
|
||||
out = _render_flash(FLASH_FULL)
|
||||
assert "**App Launching**" in out
|
||||
assert "**Timed Waiting & Transitions**" in out
|
||||
assert "{%" not in out
|
||||
|
||||
|
||||
def test_flash_without_manage_app_drops_app_launching():
|
||||
out = _render_flash(FLASH_FULL - {"manage_app"})
|
||||
assert "**App Launching**" not in out
|
||||
assert not _tool_refs(out, "manage_app")
|
||||
# The neighbouring numbered rule survives.
|
||||
assert "**Screenshot Pruning Awareness**" in out
|
||||
|
||||
|
||||
def test_flash_without_wait_for_delay_drops_timed_waiting():
|
||||
out = _render_flash(FLASH_FULL - {"wait_for_delay"})
|
||||
assert "**Timed Waiting & Transitions**" not in out
|
||||
assert not _tool_refs(out, "wait_for_delay")
|
||||
assert "**State Drift & Progression Tolerance**" in out
|
||||
|
||||
|
||||
# --- FailureAnalyzer strategy gating -------------------------------------------------
|
||||
|
||||
|
||||
def _analyzer() -> FailureAnalyzer:
|
||||
return FailureAnalyzer.__new__(FailureAnalyzer) # _select_strategy needs no ctx
|
||||
|
||||
|
||||
def test_strategy_selection_unchanged_without_gating():
|
||||
analyzer = _analyzer()
|
||||
assert isinstance(
|
||||
analyzer._select_strategy(ValidationErrorCategory.TARGET_DISAPPEARED),
|
||||
TargetDisappearedStrategy,
|
||||
)
|
||||
assert isinstance(
|
||||
analyzer._select_strategy(ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED),
|
||||
PixelTargetDisappearedStrategy,
|
||||
)
|
||||
|
||||
|
||||
def test_strategy_falls_back_when_required_tool_missing():
|
||||
"""Strategy + prompt are one unit: no click_sequence, no disappeared-prompts."""
|
||||
analyzer = _analyzer()
|
||||
strategy = analyzer._select_strategy(
|
||||
ValidationErrorCategory.TARGET_DISAPPEARED,
|
||||
available_tools=frozenset({"click", "swipe"}),
|
||||
)
|
||||
assert type(strategy) is FailureAnalysisStrategy
|
||||
assert strategy.get_prompt_template_name() == "failure_analyzer.md"
|
||||
|
||||
|
||||
def test_strategy_kept_when_required_tool_present():
|
||||
analyzer = _analyzer()
|
||||
strategy = analyzer._select_strategy(
|
||||
ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED,
|
||||
available_tools=frozenset({"click_sequence"}),
|
||||
)
|
||||
assert isinstance(strategy, PixelTargetDisappearedStrategy)
|
||||
+112
-251
@@ -12,13 +12,25 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""ValidatorNode tests over the in-process unified action session.
|
||||
|
||||
The Validator no longer spawns a stdio MCP subprocess; it calls a shared
|
||||
``ActionSession`` whose device actions return structured ``ActionResult`` objects.
|
||||
``FakeActionSession`` below stands in for it: ``action_handler`` decides each
|
||||
action's ``(ok, message)``, ``screenshot`` feeds the polling loop, and ``hierarchy``
|
||||
feeds the safety-net XML validation.
|
||||
"""
|
||||
|
||||
import json
|
||||
from unittest.mock import AsyncMock, MagicMock, Mock, patch
|
||||
|
||||
from langchain_core.messages import AIMessage
|
||||
import pytest
|
||||
|
||||
from artemis.agents.validator.failure_analyzer import ValidationErrorCategory
|
||||
from artemis.agents.validator.validator import ValidatorNode
|
||||
from artemis.context import ArtemisContext
|
||||
import pytest
|
||||
from artemis.mcp.action_types import ActionCode, ActionResult
|
||||
|
||||
|
||||
class DummyState:
|
||||
@@ -38,6 +50,46 @@ class DummyState:
|
||||
return update
|
||||
|
||||
|
||||
class FakeActionSession:
|
||||
"""Test double for artemis.mcp.action_session.ActionSession."""
|
||||
|
||||
def __init__(self):
|
||||
self.started = True
|
||||
self.calls: list[tuple[str, dict]] = []
|
||||
# (name, args) -> (ok, message)
|
||||
self.action_handler = lambda name, args: (True, "")
|
||||
# str | callable | Exception
|
||||
self.screenshot = "ZHVtbXlfZGF0YQ==" # b64("dummy_data")
|
||||
self.hierarchy: list | Exception = []
|
||||
|
||||
async def call(self, name, args):
|
||||
self.calls.append((name, args))
|
||||
ok, msg = self.action_handler(name, args)
|
||||
return ActionResult(
|
||||
ok=ok,
|
||||
code=ActionCode.OK if ok else ActionCode.DEVICE_ERROR,
|
||||
action=name,
|
||||
message=msg,
|
||||
)
|
||||
|
||||
async def screenshot_b64(self, timeout=None):
|
||||
shot = self.screenshot
|
||||
if isinstance(shot, Exception):
|
||||
raise shot
|
||||
return shot() if callable(shot) else shot
|
||||
|
||||
async def ui_hierarchy(self, timeout=None):
|
||||
if isinstance(self.hierarchy, Exception):
|
||||
raise self.hierarchy
|
||||
return self.hierarchy
|
||||
|
||||
async def aclose(self):
|
||||
pass
|
||||
|
||||
def calls_for(self, name: str) -> list[dict]:
|
||||
return [args for n, args in self.calls if n == name]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_context(tmp_path):
|
||||
ctx = Mock(spec=ArtemisContext)
|
||||
@@ -45,6 +97,9 @@ def mock_context(tmp_path):
|
||||
ctx.data_engine = Mock()
|
||||
ctx.data_engine.base_dir = tmp_path
|
||||
ctx.data_engine.get_relative_time.return_value = 1.0
|
||||
ctx.device = Mock()
|
||||
ctx.device.device_width = 1080
|
||||
ctx.device.device_height = 2400
|
||||
return ctx
|
||||
|
||||
|
||||
@@ -57,31 +112,18 @@ def temp_screenshot(tmp_path):
|
||||
|
||||
@pytest.fixture
|
||||
def mock_mcp():
|
||||
with (
|
||||
patch("mcp.client.stdio.stdio_client") as mock_stdio,
|
||||
patch("artemis.agents.validator.validator.ClientSession") as mock_session_cls,
|
||||
):
|
||||
mock_ctx = AsyncMock()
|
||||
mock_ctx.__aenter__.return_value = (Mock(), Mock())
|
||||
mock_stdio.return_value = mock_ctx
|
||||
fake = FakeActionSession()
|
||||
|
||||
mock_session = AsyncMock()
|
||||
mock_session.initialize = AsyncMock()
|
||||
mock_session_cls.return_value = mock_session
|
||||
async def _get_session(ctx, actuator=None):
|
||||
return fake
|
||||
|
||||
yield mock_session
|
||||
with patch("artemis.agents.validator.validator.get_action_session", _get_session):
|
||||
yield fake
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_success(mock_mcp, mock_context, temp_screenshot):
|
||||
"""Test that ValidatorNode executes actions successfully."""
|
||||
# Mock MCP calls: screenshot, tap, screenshot
|
||||
mock_mcp.call_tool.side_effect = [
|
||||
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # pre-screenshot
|
||||
Mock(content=[Mock(text="Success")]), # tap action
|
||||
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # post-screenshot
|
||||
]
|
||||
|
||||
decisions = json.dumps([{"action": "tap", "coordinates": [105, 205]}])
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
|
||||
@@ -95,78 +137,23 @@ async def test_validator_success(mock_mcp, mock_context, temp_screenshot):
|
||||
assert len(report["execution"]) == 1
|
||||
assert report["execution"][0]["action"] == "tap"
|
||||
assert "attempts" not in report["execution"][0]
|
||||
# The Operator's tap verb was translated to the canonical click tool.
|
||||
assert len(mock_mcp.calls_for("click")) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failed_mcp_handshake_exits_contexts_and_disables_child_awake_policy():
|
||||
"""A handshake failure must not leave AnyIO cancellation scopes behind."""
|
||||
events = []
|
||||
|
||||
class FakeClientContext:
|
||||
async def __aenter__(self):
|
||||
events.append("client_enter")
|
||||
return Mock(), Mock()
|
||||
|
||||
async def __aexit__(self, exc_type, exc, traceback):
|
||||
events.append("client_exit")
|
||||
|
||||
class FakeSession:
|
||||
async def __aenter__(self):
|
||||
events.append("session_enter")
|
||||
return self
|
||||
|
||||
async def __aexit__(self, exc_type, exc, traceback):
|
||||
events.append("session_exit")
|
||||
|
||||
async def initialize(self):
|
||||
raise TimeoutError("handshake timed out")
|
||||
|
||||
ctx = Mock(spec=ArtemisContext)
|
||||
ctx.mcp_client_ctx = None
|
||||
ctx.mcp_session = None
|
||||
fake_client_ctx = FakeClientContext()
|
||||
fake_session = FakeSession()
|
||||
captured_parameters = []
|
||||
|
||||
def fake_stdio_client(parameters):
|
||||
captured_parameters.append(parameters)
|
||||
return fake_client_ctx
|
||||
|
||||
node = ValidatorNode(ctx)
|
||||
with (
|
||||
patch("artemis.agents.validator.validator.stdio_client", fake_stdio_client),
|
||||
patch("artemis.agents.validator.validator.ClientSession", return_value=fake_session),
|
||||
patch("artemis.agents.validator.validator.AdbClient") as adb_client,
|
||||
patch.object(node, "_kill_mcp_server_instantly") as emergency_kill,
|
||||
):
|
||||
adb_client.return_value.device_list.return_value = []
|
||||
result = await node._get_mcp_session()
|
||||
|
||||
assert result is None
|
||||
assert events == ["client_enter", "session_enter", "session_exit", "client_exit"]
|
||||
assert ctx.mcp_session is None
|
||||
assert ctx.mcp_client_ctx is None
|
||||
assert captured_parameters[0].env["ARTEMIS_KEEP_DEVICE_AWAKE"] == "false"
|
||||
emergency_kill.assert_not_called()
|
||||
|
||||
# NOTE: upstream's test_failed_mcp_handshake_exits_contexts_and_disables_child_awake_policy
|
||||
# guarded the stdio-subprocess handshake teardown, machinery this migration removed
|
||||
# entirely. Its hazard class (leaked AnyIO cancel scopes) is now pinned by
|
||||
# tests/unit/mcp/test_action_session.py against the in-process ActionSession.
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_failure_analysis(mock_mcp, mock_context, temp_screenshot):
|
||||
"""Test that ValidatorNode triggers failure analysis on error and handles
|
||||
cannot_fix via tool.
|
||||
"""
|
||||
tap_count = 0
|
||||
|
||||
def mock_call_tool(name, args):
|
||||
nonlocal tap_count
|
||||
if name == "take_screenshot":
|
||||
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
|
||||
if name == "tap":
|
||||
tap_count += 1
|
||||
return Mock(content=[Mock(text="Error: Element not found")])
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
|
||||
mock_mcp.call_tool.side_effect = mock_call_tool
|
||||
mock_mcp.action_handler = lambda name, args: (
|
||||
(False, "Error: Element not found") if name == "click" else (True, "")
|
||||
)
|
||||
|
||||
decisions = json.dumps([{"action": "tap", "coordinates": [105, 205]}])
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
@@ -203,18 +190,15 @@ async def test_validator_repair_success(mock_mcp, mock_context, temp_screenshot)
|
||||
"""
|
||||
tap_count = 0
|
||||
|
||||
def mock_call_tool(name, args):
|
||||
def handler(name, args):
|
||||
nonlocal tap_count
|
||||
if name == "take_screenshot":
|
||||
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
|
||||
if name == "tap":
|
||||
if name == "click":
|
||||
tap_count += 1
|
||||
if tap_count <= 2: # First tap fails
|
||||
return Mock(content=[Mock(text="Error: Element not found")])
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
if tap_count <= 2: # First action fails on both validator attempts
|
||||
return False, "Error: Element not found"
|
||||
return True, ""
|
||||
|
||||
mock_mcp.call_tool.side_effect = mock_call_tool
|
||||
mock_mcp.action_handler = handler
|
||||
|
||||
# Two actions planned initially
|
||||
decisions = json.dumps(
|
||||
@@ -254,12 +238,6 @@ async def test_validator_repair_success(mock_mcp, mock_context, temp_screenshot)
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_wait_for_delay(mock_mcp, mock_context, temp_screenshot):
|
||||
"""Test that wait_for_delay executes successfully."""
|
||||
|
||||
def mock_call(name, args):
|
||||
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
|
||||
|
||||
mock_mcp.call_tool.side_effect = mock_call
|
||||
|
||||
decisions = json.dumps([{"action": "wait_for_delay", "time_in_ms": 10}])
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
|
||||
@@ -272,19 +250,13 @@ async def test_validator_wait_for_delay(mock_mcp, mock_context, temp_screenshot)
|
||||
assert "execution" in report
|
||||
assert len(report["execution"]) == 1
|
||||
assert report["execution"][0]["action"] == "wait_for_delay"
|
||||
# wait_for_delay is a pure client-side sleep, never a session call.
|
||||
assert mock_mcp.calls_for("wait_for_delay") == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_focus_and_clear_text_no_ui_change(mock_mcp, mock_context, temp_screenshot):
|
||||
"""Test that focus_and_clear_text executes successfully even if no UI change is detected."""
|
||||
|
||||
def mock_call(name, args):
|
||||
if name == "focus_and_clear_text":
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
|
||||
|
||||
mock_mcp.call_tool.side_effect = mock_call
|
||||
|
||||
decisions = json.dumps([{"action": "focus_and_clear_text", "coordinates": [400, 500]}])
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
|
||||
@@ -297,6 +269,7 @@ async def test_validator_focus_and_clear_text_no_ui_change(mock_mcp, mock_contex
|
||||
assert "execution" in report
|
||||
assert len(report["execution"]) == 1
|
||||
assert report["execution"][0]["action"] == "focus_and_clear_text"
|
||||
assert len(mock_mcp.calls_for("focus_and_clear_text")) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -304,16 +277,6 @@ async def test_validator_silent_failure_treated_as_success(mock_mcp, mock_contex
|
||||
"""Test that silent failure (exec succeeds but no UI change) is treated as
|
||||
success and does not trigger FailureAnalyzer.
|
||||
"""
|
||||
|
||||
def mock_call_tool(name, args):
|
||||
if name == "take_screenshot":
|
||||
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
|
||||
if name == "tap":
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
|
||||
mock_mcp.call_tool.side_effect = mock_call_tool
|
||||
|
||||
decisions = json.dumps([{"action": "tap", "coordinates": [105, 205]}])
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
|
||||
@@ -344,13 +307,7 @@ async def test_validator_records_skipped_actions_on_cannot_fix(
|
||||
mock_mcp, mock_context, temp_screenshot
|
||||
):
|
||||
"""Test that ValidatorNode marks unexecuted actions as Skipped when repair fails."""
|
||||
|
||||
def mock_call_tool(name, args):
|
||||
if name == "take_screenshot":
|
||||
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
|
||||
return Mock(content=[Mock(text="Error: Connection lost")])
|
||||
|
||||
mock_mcp.call_tool.side_effect = mock_call_tool
|
||||
mock_mcp.action_handler = lambda name, args: (False, "Error: Connection lost")
|
||||
|
||||
# Two actions planned
|
||||
decisions = json.dumps(
|
||||
@@ -399,7 +356,7 @@ async def test_validator_pre_execution_validation_and_repair(
|
||||
"""Test that pre-execution validation fails when element changes, and is
|
||||
repaired by FailureAnalyzer.
|
||||
"""
|
||||
mock_live_elements = [
|
||||
mock_mcp.hierarchy = [
|
||||
{
|
||||
"text": "Sign Up",
|
||||
"bounds": "[100,200][300,300]",
|
||||
@@ -407,17 +364,6 @@ async def test_validator_pre_execution_validation_and_repair(
|
||||
}
|
||||
]
|
||||
|
||||
def mock_call_tool(name, args):
|
||||
if name == "take_screenshot":
|
||||
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
|
||||
if name == "get_ui_hierarchy":
|
||||
return Mock(content=[Mock(text=json.dumps(mock_live_elements))])
|
||||
if name == "tap":
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
|
||||
mock_mcp.call_tool.side_effect = mock_call_tool
|
||||
|
||||
decisions = json.dumps(
|
||||
[
|
||||
{
|
||||
@@ -477,7 +423,7 @@ async def test_validator_pre_execution_validation_self_healing(
|
||||
mock_mcp, mock_context, temp_screenshot
|
||||
):
|
||||
"""Test that pre-execution validation succeeds and self-heals when element shifts slightly."""
|
||||
mock_live_elements = [
|
||||
mock_mcp.hierarchy = [
|
||||
{
|
||||
"text": "Login",
|
||||
"bounds": "[100,220][300,320]",
|
||||
@@ -485,21 +431,6 @@ async def test_validator_pre_execution_validation_self_healing(
|
||||
}
|
||||
]
|
||||
|
||||
tapped_coordinates = None
|
||||
|
||||
def mock_call_tool(name, args):
|
||||
nonlocal tapped_coordinates
|
||||
if name == "take_screenshot":
|
||||
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
|
||||
if name == "get_ui_hierarchy":
|
||||
return Mock(content=[Mock(text=json.dumps(mock_live_elements))])
|
||||
if name == "tap":
|
||||
tapped_coordinates = args.get("coordinates")
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
|
||||
mock_mcp.call_tool.side_effect = mock_call_tool
|
||||
|
||||
decisions = json.dumps(
|
||||
[
|
||||
{
|
||||
@@ -518,8 +449,12 @@ async def test_validator_pre_execution_validation_self_healing(
|
||||
with patch("artemis.utils.image_diff.check_ui_change", return_value=True):
|
||||
await node(state)
|
||||
|
||||
# Center of [100, 220][300, 320] is [200, 270]
|
||||
assert tapped_coordinates == [200, 270]
|
||||
# Center of [100, 220][300, 320] is pixel [200, 270]; the healed coordinates
|
||||
# travel to the canonical click tool normalized to 0-1000 on a 1080x2400 screen:
|
||||
# round(200 * 1000 / 1080) = 185, round(270 * 1000 / 2400) = 112.
|
||||
clicks = mock_mcp.calls_for("click")
|
||||
assert len(clicks) == 1
|
||||
assert clicks[0]["target"] == [185, 112]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -531,7 +466,7 @@ async def test_validator_pre_execution_validation_anonymous_occupant(
|
||||
"""
|
||||
# The live screen contains an anonymous clickable view covering
|
||||
# target coordinates [200, 250]
|
||||
mock_live_elements = [
|
||||
mock_mcp.hierarchy = [
|
||||
{
|
||||
"text": "",
|
||||
"bounds": "[150,200][250,300]",
|
||||
@@ -540,15 +475,6 @@ async def test_validator_pre_execution_validation_anonymous_occupant(
|
||||
}
|
||||
]
|
||||
|
||||
def mock_call_tool(name, args):
|
||||
if name == "take_screenshot":
|
||||
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
|
||||
if name == "get_ui_hierarchy":
|
||||
return Mock(content=[Mock(text=json.dumps(mock_live_elements))])
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
|
||||
mock_mcp.call_tool.side_effect = mock_call_tool
|
||||
|
||||
decisions = json.dumps(
|
||||
[
|
||||
{
|
||||
@@ -596,12 +522,6 @@ async def test_validator_pre_execution_validation_anonymous_occupant(
|
||||
@pytest.mark.asyncio
|
||||
async def test_validator_pixel_validation_success(mock_mcp, mock_context, temp_screenshot):
|
||||
"""Test that the pixel safety net succeeds when Gemini reports target is present."""
|
||||
mock_mcp.call_tool.side_effect = [
|
||||
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # live screenshot for pixel check
|
||||
Mock(content=[Mock(text="Success")]), # tap action
|
||||
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # post-screenshot
|
||||
]
|
||||
|
||||
decisions = json.dumps([{"action": "tap", "coordinates": [100, 200]}])
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
|
||||
@@ -645,14 +565,6 @@ async def test_validator_pixel_validation_failure(mock_mcp, mock_context, temp_s
|
||||
"""Test that the pixel safety net fails and triggers FailureAnalyzer
|
||||
when Gemini reports target is missing.
|
||||
"""
|
||||
|
||||
def mock_call_tool(name, args):
|
||||
if name == "take_screenshot":
|
||||
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
|
||||
mock_mcp.call_tool.side_effect = mock_call_tool
|
||||
|
||||
decisions = json.dumps([{"action": "tap", "coordinates": [100, 200]}])
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
|
||||
@@ -697,23 +609,15 @@ async def test_validator_pixel_validation_failure(mock_mcp, mock_context, temp_s
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("artemis.agents.validator.validator.find_package")
|
||||
@patch("artemis.utils.app_launch_utils.launch_app_with_retries")
|
||||
async def test_validator_launch_app_passes_use_fallback_mcp(
|
||||
mock_launch_app, mock_find_package, mock_mcp, mock_context, temp_screenshot
|
||||
async def test_validator_launch_app_routes_through_manage_app(
|
||||
mock_mcp, mock_context, temp_screenshot
|
||||
):
|
||||
"""Test that ValidatorNode calls find_package with use_fallback=False
|
||||
for launch_app action in MCP mode.
|
||||
"""Test that a launch_app action item routes to the canonical manage_app tool.
|
||||
|
||||
Package resolution (find_package with use_fallback=False) now happens inside the
|
||||
actuator's manage_app implementation and is covered by the failure-analyzer tool
|
||||
tests; the Validator's contract is the translation and single dispatch.
|
||||
"""
|
||||
mock_find_package.return_value = "com.example.app"
|
||||
mock_launch_app.return_value = (True, None)
|
||||
|
||||
# Mock MCP calls: screenshot (pre), screenshot (post)
|
||||
mock_mcp.call_tool.side_effect = [
|
||||
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # pre-screenshot
|
||||
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # post-screenshot
|
||||
]
|
||||
|
||||
decisions = json.dumps([{"action": "launch_app", "app_name": "My App"}])
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
|
||||
@@ -721,27 +625,17 @@ async def test_validator_launch_app_passes_use_fallback_mcp(
|
||||
node = ValidatorNode(mock_context)
|
||||
result = await node(state)
|
||||
|
||||
mock_find_package.assert_called_once_with(mock_context, "My App", use_fallback=False)
|
||||
mock_launch_app.assert_called_once_with(mock_context, "com.example.app")
|
||||
manage_calls = mock_mcp.calls_for("manage_app")
|
||||
assert manage_calls == [{"action": "launch", "app_name": "My App"}]
|
||||
assert "last_execution_result" in result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@patch("artemis.agents.validator.validator.find_package")
|
||||
@patch("artemis.utils.app_launch_utils.launch_app_with_retries")
|
||||
async def test_validator_launch_app_failure_no_retry(
|
||||
mock_launch_app, mock_find_package, mock_mcp, mock_context, temp_screenshot
|
||||
):
|
||||
async def test_validator_launch_app_failure_no_retry(mock_mcp, mock_context, temp_screenshot):
|
||||
"""Test that ValidatorNode does NOT retry launch_app action on failure."""
|
||||
mock_find_package.return_value = "com.example.app"
|
||||
mock_launch_app.return_value = (False, "Error: Force close")
|
||||
|
||||
def mock_call_tool(name, args):
|
||||
if name == "take_screenshot":
|
||||
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
|
||||
mock_mcp.call_tool.side_effect = mock_call_tool
|
||||
mock_mcp.action_handler = lambda name, args: (
|
||||
(False, "Error: Force close") if name == "manage_app" else (True, "")
|
||||
)
|
||||
|
||||
decisions = json.dumps([{"action": "launch_app", "app_name": "My App"}])
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
@@ -760,13 +654,9 @@ async def test_validator_launch_app_failure_no_retry(
|
||||
node = ValidatorNode(mock_context)
|
||||
result = await node(state)
|
||||
|
||||
# Verify find_package was called only once
|
||||
mock_find_package.assert_called_once_with(mock_context, "My App", use_fallback=False)
|
||||
# Verify launch_app_with_retries was called only once
|
||||
# (no retry by ValidatorNode itself since it relies on launch_app_with_retries)
|
||||
mock_launch_app.assert_called_once_with(mock_context, "com.example.app")
|
||||
# launch_app allows a single attempt -- no validator-level retry.
|
||||
assert len(mock_mcp.calls_for("manage_app")) == 1
|
||||
|
||||
# Verify attempts list has only 1 element
|
||||
assert "last_execution_result" in result
|
||||
report = result["last_execution_result"]
|
||||
assert len(report["execution"]) == 1
|
||||
@@ -783,7 +673,7 @@ async def test_validator_pre_execution_validation_disappeared_not_shifted(
|
||||
"""
|
||||
# Live hierarchy only contains a giant background/container
|
||||
# containing the text "Sign Up"
|
||||
mock_live_elements = [
|
||||
mock_mcp.hierarchy = [
|
||||
{
|
||||
"text": "Sign Up",
|
||||
"bounds": "[0,0][1080,2400]",
|
||||
@@ -791,15 +681,6 @@ async def test_validator_pre_execution_validation_disappeared_not_shifted(
|
||||
}
|
||||
]
|
||||
|
||||
def mock_call_tool(name, args):
|
||||
if name == "take_screenshot":
|
||||
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
|
||||
if name == "get_ui_hierarchy":
|
||||
return Mock(content=[Mock(text=json.dumps(mock_live_elements))])
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
|
||||
mock_mcp.call_tool.side_effect = mock_call_tool
|
||||
|
||||
decisions = json.dumps(
|
||||
[
|
||||
{
|
||||
@@ -850,12 +731,6 @@ async def test_validator_pre_execution_validation_ocr_direct_to_pixel(
|
||||
"""Test that OCR-derived elements bypass XML validation and
|
||||
directly use pixel-based validation.
|
||||
"""
|
||||
mock_mcp.call_tool.side_effect = [
|
||||
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # live screenshot for pixel check
|
||||
Mock(content=[Mock(text="Success")]), # tap action
|
||||
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # post-screenshot
|
||||
]
|
||||
|
||||
decisions = json.dumps(
|
||||
[
|
||||
{
|
||||
@@ -905,7 +780,7 @@ async def test_validator_pre_execution_xml_failure_not_overridden_when_pixel_byp
|
||||
"""Test that when XML validation fails AND Pixel validation is bypassed
|
||||
(PIXEL_BYPASSED), the XML failure is NOT overridden.
|
||||
"""
|
||||
mock_live_elements = [
|
||||
mock_mcp.hierarchy = [
|
||||
{
|
||||
"text": "Sign Up",
|
||||
"bounds": "[100,200][300,300]",
|
||||
@@ -913,15 +788,6 @@ async def test_validator_pre_execution_xml_failure_not_overridden_when_pixel_byp
|
||||
}
|
||||
]
|
||||
|
||||
def mock_call_tool(name, args):
|
||||
if name == "take_screenshot":
|
||||
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
|
||||
if name == "get_ui_hierarchy":
|
||||
return Mock(content=[Mock(text=json.dumps(mock_live_elements))])
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
|
||||
mock_mcp.call_tool.side_effect = mock_call_tool
|
||||
|
||||
decisions = json.dumps(
|
||||
[
|
||||
{
|
||||
@@ -970,17 +836,13 @@ async def test_validator_pre_execution_xml_failure_not_overridden_when_pixel_byp
|
||||
async def test_validator_failure_screenshot_mcp_error_handled_safely(
|
||||
mock_mcp, mock_context, temp_screenshot
|
||||
):
|
||||
"""Test that when take_screenshot fails on MCP server and returns an error text,
|
||||
the ValidatorNode does not crash with base64 decoding error.
|
||||
"""Test that when take_screenshot fails on the session, the ValidatorNode does
|
||||
not crash -- the failure screenshot is simply absent.
|
||||
"""
|
||||
def mock_call_tool(name, args):
|
||||
if name == "take_screenshot":
|
||||
return Mock(content=[Mock(text="Error: ADB connection lost")])
|
||||
if name == "tap":
|
||||
return Mock(content=[Mock(text="Error: Target not found")])
|
||||
return Mock(content=[Mock(text="Success")])
|
||||
|
||||
mock_mcp.call_tool.side_effect = mock_call_tool
|
||||
mock_mcp.screenshot = RuntimeError("Error: ADB connection lost")
|
||||
mock_mcp.action_handler = lambda name, args: (
|
||||
(False, "Error: Target not found") if name == "click" else (True, "")
|
||||
)
|
||||
|
||||
decisions = json.dumps([{"action": "tap", "coordinates": [105, 205]}])
|
||||
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
|
||||
@@ -1002,4 +864,3 @@ async def test_validator_failure_screenshot_mcp_error_handled_safely(
|
||||
assert report["status"] == "failed"
|
||||
assert len(report["execution"]) == 1
|
||||
assert report["execution"][0]["action"] == "tap"
|
||||
|
||||
@@ -55,7 +55,9 @@ def test_darwin_platform_paths():
|
||||
"""Verify macOS platform paths adhere to Apple Library standards."""
|
||||
darwin_p = DarwinPlatform()
|
||||
assert darwin_p.os_type == OSType.MACOS
|
||||
assert "Library/Application Support" in str(darwin_p.paths.config_dir)
|
||||
# Compare path components so the assertion holds regardless of the host
|
||||
# OS path separator used by str().
|
||||
assert darwin_p.paths.config_dir.parts[-3:] == ("Library", "Application Support", "artemis")
|
||||
assert darwin_p.process.path_separator == ":"
|
||||
|
||||
|
||||
|
||||
@@ -392,10 +392,12 @@ async def test_validator_pre_execution_loop_reverted_to_exact_safety_contract(
|
||||
mock_artemis_ctx.data_engine = MagicMock()
|
||||
|
||||
node = ValidatorNode(mock_artemis_ctx)
|
||||
# The Validator now talks to the in-process unified action session.
|
||||
session = MagicMock()
|
||||
session.call_tool = AsyncMock(
|
||||
return_value=MagicMock(content=[MagicMock(text="ZHVtbXlfZGF0YQ==")])
|
||||
)
|
||||
session.started = True
|
||||
session.screenshot_b64 = AsyncMock(return_value="ZHVtbXlfZGF0YQ==")
|
||||
session.ui_hierarchy = AsyncMock(return_value=[])
|
||||
session.call = AsyncMock()
|
||||
|
||||
decisions = json.dumps(
|
||||
[
|
||||
@@ -421,6 +423,10 @@ async def test_validator_pre_execution_loop_reverted_to_exact_safety_contract(
|
||||
)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"artemis.agents.validator.validator.get_action_session",
|
||||
AsyncMock(return_value=session),
|
||||
),
|
||||
patch.object(
|
||||
ValidatorNode, "_get_initial_screenshot", new_callable=AsyncMock
|
||||
) as mock_get_screen,
|
||||
|
||||
@@ -0,0 +1,294 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Prompt <-> manifest consistency and actuator contract tests.
|
||||
|
||||
The scan below re-derives the manifest classification from the prompt sources, so
|
||||
adding a tool name to a prompt without classifying it in
|
||||
``artemis/mcp/action_manifest.py`` fails here rather than at runtime.
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.core.tool_declaration import ToolDeclaration
|
||||
from artemis.mcp.action_manifest import (
|
||||
ActuatorContractError,
|
||||
AGENT_ROLES,
|
||||
ALL_KNOWN_TOOLS,
|
||||
BACKEND_INDEPENDENT_TOOLS,
|
||||
DEVICE_ACTIONS,
|
||||
ExtensionTool,
|
||||
INTERNAL_ACTIONS,
|
||||
OPTIONAL_ACTIONS,
|
||||
REQUIRED_ACTIONS,
|
||||
available_device_actions,
|
||||
filter_declarations,
|
||||
validate_actuator,
|
||||
)
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
|
||||
# Prompt sources belonging to the three agents this manifest governs (plus the
|
||||
# Validator's safety net). Other agents (log_analyzer, image_processor, ...) have their
|
||||
# own tool universes and are deliberately out of scope.
|
||||
SCOPED_PROMPT_FILES = [
|
||||
"artemis/agents/flash/flash_runner.md",
|
||||
"artemis/agents/validator/failure_analyzer.md",
|
||||
"artemis/agents/validator/target_disappeared_analyzer.md",
|
||||
"artemis/agents/validator/pixel_target_disappeared_analyzer.md",
|
||||
"artemis/agents/validator/pixel_safety_net.md",
|
||||
]
|
||||
|
||||
# Backtick-quoted identifiers in scoped prompts that are not tools: parameter names and
|
||||
# note keys. Anything new landing here should be a conscious decision.
|
||||
NON_TOOL_IDENTIFIERS = frozenset({"analysis", "task_plan"})
|
||||
|
||||
_TOOL_REF = re.compile(r"(?:`([a-z][a-z0-9_]{2,})`|\b([a-z][a-z0-9_]{2,})\()")
|
||||
|
||||
|
||||
def _scoped_prompt_sources() -> dict[str, str]:
|
||||
"""Collects all in-scope prompt texts, keyed by a short source label."""
|
||||
sources: dict[str, str] = {}
|
||||
for rel in SCOPED_PROMPT_FILES:
|
||||
path = REPO_ROOT / rel
|
||||
sources[path.name] = path.read_text(encoding="utf-8")
|
||||
|
||||
operator_json = REPO_ROOT / "artemis/agents/operator/operator.json"
|
||||
for key, text in json.loads(operator_json.read_text(encoding="utf-8")).items():
|
||||
sources[f"operator.json:{key}"] = text
|
||||
|
||||
# Only the prompt string constants from prompts.py -- scanning the module as text
|
||||
# would pick up Python code (`len(`, `getattr(` ...).
|
||||
from artemis.agents.operator import prompts as operator_prompts
|
||||
|
||||
for name in dir(operator_prompts):
|
||||
value = getattr(operator_prompts, name)
|
||||
if name.startswith("_") and isinstance(value, str) and len(value) > 80:
|
||||
sources[f"prompts.py:{name}"] = value
|
||||
|
||||
return sources
|
||||
|
||||
|
||||
def _referenced_tools() -> dict[str, set[str]]:
|
||||
"""Returns {identifier: source labels} for every tool-shaped reference in scope."""
|
||||
found: dict[str, set[str]] = {}
|
||||
for label, text in _scoped_prompt_sources().items():
|
||||
for match in _TOOL_REF.finditer(text):
|
||||
name = match.group(1) or match.group(2)
|
||||
found.setdefault(name, set()).add(label)
|
||||
return found
|
||||
|
||||
|
||||
# --- Prompt <-> manifest invariants --------------------------------------------------
|
||||
|
||||
|
||||
def test_scan_finds_prompt_sources():
|
||||
"""The scan must cover all expected sources; silent scope loss would void the suite."""
|
||||
sources = _scoped_prompt_sources()
|
||||
assert "flash_runner.md" in sources
|
||||
assert "operator.json:main_template" in sources
|
||||
assert "operator.json:troubleshooter_template" in sources
|
||||
assert any(label.startswith("prompts.py:") for label in sources)
|
||||
|
||||
|
||||
def test_every_prompt_tool_reference_is_classified():
|
||||
"""Every tool referenced by an in-scope prompt must be known to the manifest."""
|
||||
unclassified = {
|
||||
name: sorted(labels)
|
||||
for name, labels in _referenced_tools().items()
|
||||
if name not in ALL_KNOWN_TOOLS and name not in NON_TOOL_IDENTIFIERS
|
||||
}
|
||||
assert not unclassified, (
|
||||
"Prompts reference tools the manifest does not classify. Add each to the "
|
||||
f"appropriate set in artemis/mcp/action_manifest.py: {unclassified}"
|
||||
)
|
||||
|
||||
|
||||
def test_required_actions_are_structurally_referenced():
|
||||
"""A REQUIRED action must actually appear in a prompt; otherwise demote it."""
|
||||
referenced = set(_referenced_tools())
|
||||
unreferenced = REQUIRED_ACTIONS - referenced
|
||||
assert not unreferenced, (
|
||||
f"REQUIRED_ACTIONS not referenced by any prompt: {sorted(unreferenced)}. "
|
||||
"Required status exists only to protect structural prompt dependencies."
|
||||
)
|
||||
|
||||
|
||||
def test_internal_actions_never_appear_in_prompts():
|
||||
"""INTERNAL actions are adapter plumbing and must stay invisible to every model."""
|
||||
leaked = INTERNAL_ACTIONS & set(_referenced_tools())
|
||||
assert not leaked, f"Internal actions leaked into prompts: {sorted(leaked)}"
|
||||
|
||||
|
||||
def test_manifest_sets_are_disjoint():
|
||||
sets = {
|
||||
"REQUIRED_ACTIONS": REQUIRED_ACTIONS,
|
||||
"OPTIONAL_ACTIONS": OPTIONAL_ACTIONS,
|
||||
"INTERNAL_ACTIONS": INTERNAL_ACTIONS,
|
||||
"BACKEND_INDEPENDENT_TOOLS": BACKEND_INDEPENDENT_TOOLS,
|
||||
}
|
||||
names = list(sets)
|
||||
for i, a in enumerate(names):
|
||||
for b in names[i + 1 :]:
|
||||
overlap = sets[a] & sets[b]
|
||||
assert not overlap, f"{a} and {b} overlap: {sorted(overlap)}"
|
||||
|
||||
|
||||
# --- Actuator contract ---------------------------------------------------------------
|
||||
|
||||
|
||||
class _StubActuator:
|
||||
"""Minimal actuator test double with configurable capabilities and extensions."""
|
||||
|
||||
def __init__(self, caps=None, exts=None):
|
||||
self._caps = frozenset(caps if caps is not None else DEVICE_ACTIONS)
|
||||
self._exts = list(exts or [])
|
||||
|
||||
def capabilities(self):
|
||||
return self._caps
|
||||
|
||||
def extensions(self):
|
||||
return self._exts
|
||||
|
||||
|
||||
def _extension(**overrides) -> ExtensionTool:
|
||||
async def _noop(**_kwargs):
|
||||
return "ok"
|
||||
|
||||
defaults = dict(
|
||||
name="calibrate_arm",
|
||||
description=(
|
||||
"Runs the robot arm's touch calibration sweep. Use before the first "
|
||||
"physical action of a session, or after any missed tap."
|
||||
),
|
||||
input_schema={"type": "object", "properties": {}},
|
||||
handler=_noop,
|
||||
targets=frozenset({"operator", "flash"}),
|
||||
)
|
||||
defaults.update(overrides)
|
||||
return ExtensionTool(**defaults)
|
||||
|
||||
|
||||
def test_full_actuator_passes_validation():
|
||||
validate_actuator(_StubActuator())
|
||||
|
||||
|
||||
def test_minimal_actuator_passes_validation():
|
||||
"""REQUIRED + INTERNAL alone is a legal backend; every optional action may be absent."""
|
||||
validate_actuator(_StubActuator(caps=REQUIRED_ACTIONS | INTERNAL_ACTIONS))
|
||||
|
||||
|
||||
def test_missing_required_action_fails_fast():
|
||||
caps = (DEVICE_ACTIONS - {"click_sequence"})
|
||||
with pytest.raises(ActuatorContractError, match="click_sequence"):
|
||||
validate_actuator(_StubActuator(caps=caps))
|
||||
|
||||
|
||||
def test_missing_internal_action_fails_fast():
|
||||
caps = DEVICE_ACTIONS - {"observe_screen"}
|
||||
with pytest.raises(ActuatorContractError, match="observe_screen"):
|
||||
validate_actuator(_StubActuator(caps=caps))
|
||||
|
||||
|
||||
def test_unknown_capability_is_rejected():
|
||||
"""Backend-specific tools must go through extensions(), not capabilities()."""
|
||||
with pytest.raises(ActuatorContractError, match="calibrate_arm"):
|
||||
validate_actuator(_StubActuator(caps=DEVICE_ACTIONS | {"calibrate_arm"}))
|
||||
|
||||
|
||||
def test_extension_name_collision_fails_fast():
|
||||
with pytest.raises(ActuatorContractError, match="collides"):
|
||||
validate_actuator(_StubActuator(exts=[_extension(name="click")]))
|
||||
|
||||
|
||||
def test_duplicate_extension_names_fail_fast():
|
||||
with pytest.raises(ActuatorContractError, match="more than once"):
|
||||
validate_actuator(_StubActuator(exts=[_extension(), _extension()]))
|
||||
|
||||
|
||||
def test_extension_with_thin_description_is_rejected():
|
||||
"""Extensions get no prompt-level teaching; the description carries the contract."""
|
||||
with pytest.raises(ActuatorContractError, match="description"):
|
||||
validate_actuator(_StubActuator(exts=[_extension(description="Calibrates.")]))
|
||||
|
||||
|
||||
def test_extension_with_unknown_target_is_rejected():
|
||||
with pytest.raises(ActuatorContractError, match="unknown agent role"):
|
||||
validate_actuator(_StubActuator(exts=[_extension(targets=frozenset({"planner"}))]))
|
||||
|
||||
|
||||
# --- Surface derivation --------------------------------------------------------------
|
||||
|
||||
|
||||
def _declarations(*names: str) -> list[ToolDeclaration]:
|
||||
return [
|
||||
ToolDeclaration(name=n, description=f"{n} tool", parameters={"type": "object"})
|
||||
for n in names
|
||||
]
|
||||
|
||||
|
||||
def test_available_device_actions_excludes_internal():
|
||||
actions = available_device_actions(_StubActuator())
|
||||
assert actions == REQUIRED_ACTIONS | OPTIONAL_ACTIONS
|
||||
assert not actions & INTERNAL_ACTIONS
|
||||
|
||||
|
||||
def test_filter_drops_unimplemented_optional_actions():
|
||||
actuator = _StubActuator(caps=DEVICE_ACTIONS - {"swipe", "manage_app"})
|
||||
decls = _declarations("click", "swipe", "manage_app", "read_note")
|
||||
kept = [d.name for d in filter_declarations(decls, actuator, agent="flash")]
|
||||
assert kept == ["click", "read_note"]
|
||||
|
||||
|
||||
def test_filter_passes_through_unclassified_agent_tools():
|
||||
"""Agent-specific tools the manifest does not know must never be gated."""
|
||||
actuator = _StubActuator(caps=REQUIRED_ACTIONS | INTERNAL_ACTIONS)
|
||||
decls = _declarations("click_sequence", "reply_to_checker", "some_bespoke_tool")
|
||||
kept = [d.name for d in filter_declarations(decls, actuator, agent="operator")]
|
||||
assert kept == ["click_sequence", "reply_to_checker", "some_bespoke_tool"]
|
||||
|
||||
|
||||
def test_filter_appends_extensions_for_targeted_agents_only():
|
||||
actuator = _StubActuator(exts=[_extension(targets=frozenset({"flash"}))])
|
||||
decls = _declarations("click")
|
||||
|
||||
flash = [d.name for d in filter_declarations(decls, actuator, agent="flash")]
|
||||
operator = [d.name for d in filter_declarations(decls, actuator, agent="operator")]
|
||||
|
||||
assert flash == ["click", "calibrate_arm"]
|
||||
assert operator == ["click"]
|
||||
|
||||
|
||||
def test_filter_extension_declaration_shape():
|
||||
"""Extension declarations must be indistinguishable in shape from known ones."""
|
||||
actuator = _StubActuator(exts=[_extension()])
|
||||
decls = filter_declarations([], actuator, agent="operator")
|
||||
assert len(decls) == 1
|
||||
ext_decl = decls[0]
|
||||
assert isinstance(ext_decl, ToolDeclaration)
|
||||
assert ext_decl.name == "calibrate_arm"
|
||||
assert ext_decl.parameters == {"type": "object", "properties": {}}
|
||||
|
||||
|
||||
def test_filter_rejects_unknown_agent_role():
|
||||
with pytest.raises(ValueError, match="Unknown agent role"):
|
||||
filter_declarations([], _StubActuator(), agent="planner")
|
||||
|
||||
|
||||
def test_all_agent_roles_are_covered():
|
||||
assert AGENT_ROLES == {"operator", "flash", "failure_analyzer", "validator"}
|
||||
@@ -0,0 +1,156 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""End-to-end tests of the unified action server over the in-memory transport."""
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.mcp.action_manifest import (
|
||||
ActuatorContractError,
|
||||
DEVICE_ACTIONS,
|
||||
INTERNAL_ACTIONS,
|
||||
REQUIRED_ACTIONS,
|
||||
ExtensionTool,
|
||||
)
|
||||
from artemis.mcp.action_server import build_action_server
|
||||
from artemis.mcp.action_session import ActionSession
|
||||
from artemis.mcp.actuators import MockActuator
|
||||
|
||||
pytestmark = pytest.mark.asyncio
|
||||
|
||||
|
||||
async def _session_for(actuator) -> ActionSession:
|
||||
return await ActionSession(build_action_server(actuator)).start()
|
||||
|
||||
|
||||
async def test_click_end_to_end():
|
||||
actuator = MockActuator(width=1000, height=2000)
|
||||
session = await _session_for(actuator)
|
||||
try:
|
||||
res = await session.call("click", {"target": [500, 500]})
|
||||
assert res.ok
|
||||
assert res.message == "Clicked at [500, 500] (normalized) successfully."
|
||||
assert actuator.action_history[-1]["action"] == "tap"
|
||||
assert actuator.action_history[-1]["x"] == 500
|
||||
assert actuator.action_history[-1]["y"] == 1000
|
||||
finally:
|
||||
await session.aclose()
|
||||
|
||||
|
||||
async def test_device_refusal_is_not_a_protocol_error():
|
||||
"""A failed action reaches the caller as ok=False, never as an exception."""
|
||||
actuator = MockActuator()
|
||||
session = await _session_for(actuator)
|
||||
try:
|
||||
res = await session.call("press_key", {"key": ""})
|
||||
assert not res.ok
|
||||
assert res.code.value == "INVALID_ARGS"
|
||||
assert res.message == "Error executing key press ''."
|
||||
finally:
|
||||
await session.aclose()
|
||||
|
||||
|
||||
async def test_partial_actuator_hides_optional_tools():
|
||||
"""An unimplemented optional tool does not exist anywhere on the surface."""
|
||||
minimal = MockActuator(capabilities=REQUIRED_ACTIONS | INTERNAL_ACTIONS)
|
||||
session = await _session_for(minimal)
|
||||
try:
|
||||
listed = {t.name for t in (await session._session.list_tools()).tools}
|
||||
assert "click_sequence" in listed
|
||||
assert "observe_screen" in listed
|
||||
for absent in ("click", "swipe", "manage_app", "wait_for_text", "open_link"):
|
||||
assert absent not in listed
|
||||
finally:
|
||||
await session.aclose()
|
||||
|
||||
|
||||
async def test_missing_required_action_fails_at_build_time():
|
||||
broken = MockActuator(capabilities=DEVICE_ACTIONS - {"click_sequence"})
|
||||
with pytest.raises(ActuatorContractError, match="click_sequence"):
|
||||
build_action_server(broken)
|
||||
|
||||
|
||||
async def test_extension_tool_is_served_and_callable():
|
||||
async def calibrate(sweep_speed: int = 1) -> str:
|
||||
return f"calibrated at speed {sweep_speed}"
|
||||
|
||||
ext = ExtensionTool(
|
||||
name="calibrate_arm",
|
||||
description=(
|
||||
"Runs the robot arm's touch calibration sweep. Use before the first "
|
||||
"physical action of a session, or after any missed tap."
|
||||
),
|
||||
input_schema={"type": "object", "properties": {}},
|
||||
handler=calibrate,
|
||||
targets=frozenset({"flash"}),
|
||||
)
|
||||
actuator = MockActuator(extensions=[ext])
|
||||
session = await _session_for(actuator)
|
||||
try:
|
||||
listed = {t.name for t in (await session._session.list_tools()).tools}
|
||||
assert "calibrate_arm" in listed
|
||||
result = await session.call_raw("calibrate_arm", {"sweep_speed": 3})
|
||||
text = next(b.text for b in result.content if b.type == "text")
|
||||
assert text == "calibrated at speed 3"
|
||||
finally:
|
||||
await session.aclose()
|
||||
|
||||
|
||||
async def test_extension_name_collision_fails_at_build_time():
|
||||
async def bogus() -> str:
|
||||
return "x"
|
||||
|
||||
ext = ExtensionTool(
|
||||
name="click",
|
||||
description="An extension improperly reusing a canonical tool name, which "
|
||||
"must be rejected at build time.",
|
||||
input_schema={"type": "object"},
|
||||
handler=bogus,
|
||||
)
|
||||
with pytest.raises(ActuatorContractError, match="collides"):
|
||||
build_action_server(MockActuator(extensions=[ext]))
|
||||
|
||||
|
||||
async def test_unknown_tool_raises_protocol_error():
|
||||
session = await _session_for(MockActuator())
|
||||
try:
|
||||
with pytest.raises(RuntimeError, match="not_a_tool"):
|
||||
await session.call_raw("not_a_tool", {})
|
||||
finally:
|
||||
await session.aclose()
|
||||
|
||||
|
||||
async def test_screenshot_and_hierarchy_helpers():
|
||||
session = await _session_for(MockActuator(width=1000, height=2000))
|
||||
try:
|
||||
b64 = await session.screenshot_b64()
|
||||
assert isinstance(b64, str) and len(b64) > 0
|
||||
elements = await session.ui_hierarchy()
|
||||
assert elements is not None
|
||||
finally:
|
||||
await session.aclose()
|
||||
|
||||
|
||||
async def test_structured_content_carries_real_status():
|
||||
"""The wire result exposes ok/code so no consumer ever sniffs substrings again."""
|
||||
session = await _session_for(MockActuator())
|
||||
try:
|
||||
raw = await session.call_raw("click", {"target": [10, 10]})
|
||||
sc = raw.structuredContent
|
||||
assert sc["ok"] is True
|
||||
assert sc["code"] == "OK"
|
||||
assert sc["action"] == "click"
|
||||
assert raw.isError is False
|
||||
finally:
|
||||
await session.aclose()
|
||||
@@ -0,0 +1,188 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""ActionSession lifecycle tests, including the anyio same-task exit hazard."""
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.mcp.action_server import build_action_server
|
||||
from artemis.mcp.action_session import ActionSession, get_action_session
|
||||
from artemis.mcp.actuators import MockActuator
|
||||
|
||||
pytestmark = pytest.mark.asyncio
|
||||
|
||||
|
||||
async def test_start_and_close():
|
||||
session = ActionSession(build_action_server(MockActuator()))
|
||||
await session.start()
|
||||
assert session.started
|
||||
await session.aclose()
|
||||
assert not session.started
|
||||
|
||||
|
||||
async def test_close_from_different_task():
|
||||
"""The regression guard for the anyio cancel-scope same-task rule.
|
||||
|
||||
The in-memory transport's task group is entered inside the owner task; closing
|
||||
from any *other* task must not raise ``RuntimeError: Attempted to exit cancel
|
||||
scope in a different task``.
|
||||
"""
|
||||
session = ActionSession(build_action_server(MockActuator()))
|
||||
await session.start()
|
||||
|
||||
async def _closer():
|
||||
await session.aclose()
|
||||
|
||||
await asyncio.create_task(_closer())
|
||||
assert not session.started
|
||||
|
||||
|
||||
async def test_caller_timeout_does_not_brick_the_session():
|
||||
"""Regression: cancelling a caller must never corrupt the shared transport.
|
||||
|
||||
Observed live on-device: ``asyncio.wait_for(session.ui_hierarchy(), 1.0)`` around
|
||||
a slow hierarchy fetch cancelled ``call_tool`` mid-flight, corrupting the
|
||||
in-memory streams -- every later call failed with an empty-message
|
||||
``BrokenResourceError`` and the server never received another request. The owner
|
||||
request queue makes an abandoned caller harmless; this test pins that down.
|
||||
"""
|
||||
actuator = MockActuator(width=1000, height=2000)
|
||||
|
||||
async def _slow_get_ui_elements():
|
||||
await asyncio.sleep(2.0)
|
||||
return []
|
||||
|
||||
actuator.get_ui_elements = _slow_get_ui_elements
|
||||
session = await ActionSession(build_action_server(actuator)).start()
|
||||
try:
|
||||
with pytest.raises(asyncio.TimeoutError):
|
||||
await asyncio.wait_for(session.ui_hierarchy(), timeout=0.1)
|
||||
|
||||
# The session must still be fully functional afterwards.
|
||||
result = await session.call("click", {"target": [500, 500]})
|
||||
assert result.ok
|
||||
taps = [h for h in actuator.action_history if h["action"] == "tap"]
|
||||
assert len(taps) == 1
|
||||
finally:
|
||||
await session.aclose()
|
||||
|
||||
|
||||
async def test_per_call_timeout_fires_server_side_and_session_survives():
|
||||
"""A per-call ``timeout`` runs inside the server tool, never on the transport.
|
||||
|
||||
mcp 1.29's ``read_timeout_seconds`` is NOT safe on the in-memory transport: a
|
||||
late response to a timed-out request crashes the client receive loop
|
||||
(``_handle_response`` sends into the request's already-closed response stream) --
|
||||
observed live on-device as a session bricked after a single clean
|
||||
``McpError`` timeout. Server-side timeouts produce an ordinary error payload
|
||||
and no late response, so the session stays healthy by construction.
|
||||
"""
|
||||
actuator = MockActuator(width=1000, height=2000)
|
||||
|
||||
async def _slow_get_ui_elements():
|
||||
await asyncio.sleep(2.0)
|
||||
return []
|
||||
|
||||
actuator.get_ui_elements = _slow_get_ui_elements
|
||||
session = await ActionSession(build_action_server(actuator)).start()
|
||||
try:
|
||||
with pytest.raises(RuntimeError, match="timed out"):
|
||||
await session.ui_hierarchy(timeout=0.1)
|
||||
|
||||
result = await session.call("click", {"target": [500, 500]})
|
||||
assert result.ok
|
||||
finally:
|
||||
await session.aclose()
|
||||
|
||||
|
||||
async def test_transport_death_retires_session_for_rebuild():
|
||||
"""A dead transport must flip ``started`` so get_action_session rebuilds.
|
||||
|
||||
Without this, one transport failure leaves the rest of a run failing on a
|
||||
corpse (observed live: 24 minutes of empty-message ``ClosedResourceError``).
|
||||
"""
|
||||
import anyio
|
||||
|
||||
session = await ActionSession(build_action_server(MockActuator())).start()
|
||||
real_call_tool = session._session.call_tool
|
||||
|
||||
async def _dying_call_tool(name, args, **kw):
|
||||
raise anyio.ClosedResourceError()
|
||||
|
||||
session._session.call_tool = _dying_call_tool
|
||||
with pytest.raises(anyio.ClosedResourceError):
|
||||
await session.call("click", {"target": [1, 1]})
|
||||
|
||||
# Owner loop retires; the session reports itself unusable. The owner task may
|
||||
# finish with an exception group from the transport teardown -- aclose() is the
|
||||
# API that swallows that, so here we just wait for `started` to flip.
|
||||
for _ in range(50):
|
||||
if not session.started:
|
||||
break
|
||||
await asyncio.sleep(0.1)
|
||||
assert not session.started
|
||||
|
||||
class Ctx:
|
||||
action_session = session
|
||||
|
||||
fresh = await get_action_session(Ctx(), actuator=MockActuator())
|
||||
assert fresh is not session
|
||||
assert fresh.started
|
||||
await fresh.aclose()
|
||||
del real_call_tool
|
||||
|
||||
|
||||
async def test_calls_are_serialized_by_the_lock():
|
||||
"""Two concurrent calls must not interleave on the single device."""
|
||||
actuator = MockActuator(width=1000, height=2000)
|
||||
session = await ActionSession(build_action_server(actuator)).start()
|
||||
try:
|
||||
await asyncio.gather(
|
||||
session.call("click", {"target": [100, 100]}),
|
||||
session.call("click", {"target": [900, 900]}),
|
||||
)
|
||||
taps = [h for h in actuator.action_history if h["action"] == "tap"]
|
||||
assert len(taps) == 2
|
||||
finally:
|
||||
await session.aclose()
|
||||
|
||||
|
||||
async def test_call_before_start_raises():
|
||||
session = ActionSession(build_action_server(MockActuator()))
|
||||
with pytest.raises(RuntimeError, match="not started"):
|
||||
await session.call("click", {"target": [1, 1]})
|
||||
|
||||
|
||||
async def test_double_start_is_idempotent():
|
||||
session = ActionSession(build_action_server(MockActuator()))
|
||||
await session.start()
|
||||
owner = session._owner
|
||||
await session.start()
|
||||
assert session._owner is owner
|
||||
await session.aclose()
|
||||
|
||||
|
||||
async def test_get_action_session_caches_on_ctx():
|
||||
class Ctx:
|
||||
action_session = None
|
||||
|
||||
ctx = Ctx()
|
||||
actuator = MockActuator()
|
||||
session = await get_action_session(ctx, actuator=actuator)
|
||||
assert ctx.action_session is session
|
||||
again = await get_action_session(ctx)
|
||||
assert again is session
|
||||
await session.aclose()
|
||||
@@ -0,0 +1,146 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Actuator layer tests: coordinate conversion, message parity, driver dispatch."""
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.mcp.action_manifest import DEVICE_ACTIONS
|
||||
from artemis.mcp.action_types import ActionCode
|
||||
from artemis.mcp.actuators import Actuator, AdbActuator, MockActuator
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def actuator() -> MockActuator:
|
||||
return MockActuator(width=1000, height=2000)
|
||||
|
||||
|
||||
def test_mock_actuator_satisfies_protocol(actuator):
|
||||
assert isinstance(actuator, Actuator)
|
||||
assert isinstance(actuator, AdbActuator)
|
||||
assert actuator.capabilities() == DEVICE_ACTIONS
|
||||
assert actuator.extensions() == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_click_converts_normalized_to_pixels(actuator):
|
||||
res = await actuator.click(500, 500)
|
||||
assert res.ok and res.code is ActionCode.OK
|
||||
# Historical wording is part of the contract: transcripts and traces assert on it.
|
||||
assert res.message == "Clicked at [500, 500] (normalized) successfully."
|
||||
assert res.normalized_coordinates == [500, 500]
|
||||
assert actuator.action_history[-1] == {
|
||||
"action": "tap",
|
||||
"x": 500,
|
||||
"y": 1000,
|
||||
"times": 1,
|
||||
"duration_ms": 100,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_click_sequence_taps_each_point(actuator):
|
||||
res = await actuator.click_sequence([(100, 100), (900, 900)], delay_ms=1)
|
||||
assert res.ok
|
||||
assert res.message == (
|
||||
"Sequence clicked successfully: Tapped at [100, 200]; Tapped at [900, 1800]"
|
||||
)
|
||||
taps = [h for h in actuator.action_history if h["action"] == "tap"]
|
||||
assert [(t["x"], t["y"]) for t in taps] == [(100, 200), (900, 1800)]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_long_press_message_and_duration(actuator):
|
||||
res = await actuator.long_press(250, 750, duration_ms=1500)
|
||||
assert res.ok
|
||||
assert res.message == (
|
||||
"Long pressed at [250, 750] (normalized) for 1500ms successfully."
|
||||
)
|
||||
assert res.duration_ms == 1500
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_swipe_reports_normalized_endpoints(actuator):
|
||||
res = await actuator.swipe((500, 800), (500, 200), 400)
|
||||
assert res.ok
|
||||
assert res.message == "Swipe completed successfully. Swiped from [500, 800] to [500, 200]."
|
||||
assert res.normalized_coordinates == [500, 800, 500, 200]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_press_key_forwards_unknown_keycode_to_driver(actuator):
|
||||
# Arbitrary keycodes (KEYCODE_* names, numeric codes) pass through to the driver,
|
||||
# matching the historical adb_server behavior of accepting any Android key event.
|
||||
res = await actuator.press_key("KEYCODE_DPAD_DOWN")
|
||||
assert res.ok
|
||||
assert res.message == "Executed key press 'KEYCODE_DPAD_DOWN'."
|
||||
assert actuator.action_history[-1]["action"] == "press_key"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_press_key_rejects_empty_key(actuator):
|
||||
res = await actuator.press_key("")
|
||||
assert not res.ok
|
||||
assert res.code is ActionCode.INVALID_ARGS
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_input_text_append_moves_cursor_and_preserves_text(actuator, monkeypatch):
|
||||
monkeypatch.setattr("asyncio.sleep", _instant_sleep)
|
||||
res = await actuator.input_text("hello", (500, 300), clear_exist=False)
|
||||
assert res.ok
|
||||
assert res.message == "Executed typing 'hello'."
|
||||
actions = [h["action"] for h in actuator.action_history]
|
||||
# Append semantics: focus tap, cursor-to-end (KEYCODE_MOVE_END), then type without
|
||||
# clearing — matching the historical adb_server focus_and_input_text contract.
|
||||
assert actions == ["tap", "press_key", "input_text"]
|
||||
typed = actuator.action_history[-1]
|
||||
assert typed["clear_existing"] is False
|
||||
cursor = actuator.action_history[-2]
|
||||
assert cursor["key"] == "123"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_input_text_clear_erases_then_types_without_reclearing(actuator, monkeypatch):
|
||||
monkeypatch.setattr("asyncio.sleep", _instant_sleep)
|
||||
res = await actuator.input_text("hello", (500, 300), clear_exist=True)
|
||||
assert res.ok
|
||||
actions = [h["action"] for h in actuator.action_history]
|
||||
# The controller's full clear runs as a shell keyevent batch on this driver.
|
||||
assert actions == ["tap", "execute_shell", "input_text"]
|
||||
assert actuator.action_history[-1]["clear_existing"] is False
|
||||
|
||||
|
||||
async def _instant_sleep(_delay, *args, **kwargs):
|
||||
return None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wait_for_delay(actuator):
|
||||
res = await actuator.wait_for_delay(1)
|
||||
assert res.ok
|
||||
assert res.message == "Waited for 1ms successfully."
|
||||
|
||||
|
||||
def test_partial_capabilities_are_reported():
|
||||
partial = MockActuator(capabilities=frozenset({"click_sequence"}))
|
||||
assert partial.capabilities() == frozenset({"click_sequence"})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_observation_primitives(actuator):
|
||||
shot = await actuator.take_screenshot()
|
||||
assert isinstance(shot, str) and len(shot) > 0
|
||||
data = await actuator.get_screen_data()
|
||||
assert data.width == 1000 and data.height == 2000
|
||||
@@ -20,8 +20,10 @@ subprocess stdin isolation against pipe hijacking, and stdout purity.
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import queue
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
@@ -34,6 +36,31 @@ from artemis.utils.logger import get_logger
|
||||
from mcp_server.utils import device_utils, env_utils
|
||||
|
||||
|
||||
def _readline_with_timeout(pipe, timeout: float) -> str | None:
|
||||
"""Read one line from a subprocess pipe with a timeout, portably.
|
||||
|
||||
select.select only supports sockets on Windows, so poll via a reader thread
|
||||
instead of selecting on the pipe handle.
|
||||
"""
|
||||
result: queue.Queue = queue.Queue()
|
||||
|
||||
def _reader():
|
||||
try:
|
||||
result.put(pipe.readline())
|
||||
except Exception:
|
||||
result.put(b"")
|
||||
|
||||
reader = threading.Thread(target=_reader, daemon=True)
|
||||
reader.start()
|
||||
try:
|
||||
line = result.get(timeout=timeout)
|
||||
except queue.Empty:
|
||||
return None
|
||||
if not line:
|
||||
return None
|
||||
return line.decode("utf-8").strip()
|
||||
|
||||
|
||||
def test_mcp_stdio_handshake_immediate_input():
|
||||
"""Verify that MCP server over stdio responds to an immediate initialize request
|
||||
|
||||
@@ -51,6 +78,24 @@ def test_mcp_stdio_handshake_immediate_input():
|
||||
"PYTHONUNBUFFERED": "1",
|
||||
"PATH": os.environ.get("PATH", ""),
|
||||
"PYTHONPATH": project_root,
|
||||
# Windows: the interpreter's socket stack (Winsock/_overlapped),
|
||||
# Path.home(), and tempfile handling need these system variables.
|
||||
**{
|
||||
key: os.environ[key]
|
||||
for key in (
|
||||
"SYSTEMROOT",
|
||||
"SYSTEMDRIVE",
|
||||
"WINDIR",
|
||||
"TEMP",
|
||||
"TMP",
|
||||
"USERPROFILE",
|
||||
"HOMEDRIVE",
|
||||
"HOMEPATH",
|
||||
"APPDATA",
|
||||
"LOCALAPPDATA",
|
||||
)
|
||||
if key in os.environ
|
||||
},
|
||||
},
|
||||
cwd=project_root,
|
||||
)
|
||||
@@ -71,15 +116,7 @@ def test_mcp_stdio_handshake_immediate_input():
|
||||
p.stdin.flush()
|
||||
|
||||
# Read initialize response with a strict timeout
|
||||
start = time.time()
|
||||
init_resp_line = None
|
||||
while time.time() - start < 6.0:
|
||||
import select
|
||||
|
||||
r, _, _ = select.select([p.stdout], [], [], 0.2)
|
||||
if r:
|
||||
init_resp_line = p.stdout.readline().decode("utf-8").strip()
|
||||
break
|
||||
init_resp_line = _readline_with_timeout(p.stdout, 6.0)
|
||||
|
||||
assert init_resp_line is not None, "MCP server failed to respond to initialize request within 6 seconds (deadlock detected)!"
|
||||
init_data = json.loads(init_resp_line)
|
||||
@@ -93,15 +130,7 @@ def test_mcp_stdio_handshake_immediate_input():
|
||||
p.stdin.write(json.dumps(tools_req).encode("utf-8") + b"\n")
|
||||
p.stdin.flush()
|
||||
|
||||
tools_resp_line = None
|
||||
start = time.time()
|
||||
while time.time() - start < 4.0:
|
||||
import select
|
||||
|
||||
r, _, _ = select.select([p.stdout], [], [], 0.2)
|
||||
if r:
|
||||
tools_resp_line = p.stdout.readline().decode("utf-8").strip()
|
||||
break
|
||||
tools_resp_line = _readline_with_timeout(p.stdout, 4.0)
|
||||
|
||||
assert tools_resp_line is not None, "MCP server failed to respond to tools/list request!"
|
||||
tools_data = json.loads(tools_resp_line)
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Tests for the spawn watchdog guarding mobile_run_task's standalone fallback."""
|
||||
|
||||
import json
|
||||
import os
|
||||
from unittest.mock import patch
|
||||
|
||||
from mcp_server.tools import task_runner as spawn_tool
|
||||
|
||||
|
||||
def _write_status(trace_dir: str, trace_id: str, status: str) -> None:
|
||||
os.makedirs(trace_dir, exist_ok=True)
|
||||
with open(os.path.join(trace_dir, "status.json"), "w", encoding="utf-8") as f:
|
||||
json.dump({"trace_id": trace_id, "status": status}, f)
|
||||
|
||||
|
||||
def _patch_trace_dir(tmp_path):
|
||||
return patch.object(spawn_tool.trace_store, "TRACES_DIR", str(tmp_path))
|
||||
|
||||
|
||||
def test_watchdog_passes_when_runner_creates_logs(tmp_path):
|
||||
trace_id = "wd-boot-ok"
|
||||
trace_dir = tmp_path / trace_id
|
||||
_write_status(str(trace_dir), trace_id, "running")
|
||||
(trace_dir / "stdout.log").write_text("booted", encoding="utf-8")
|
||||
|
||||
with _patch_trace_dir(tmp_path), patch.object(spawn_tool, "_kill_process_tree") as kill:
|
||||
assert spawn_tool._watch_spawn(
|
||||
trace_id, pid=999999, queue_ticket="t", conversation_id=None,
|
||||
deadline_seconds=0.2, poll_interval=0.05,
|
||||
)
|
||||
kill.assert_not_called()
|
||||
|
||||
|
||||
def test_watchdog_skips_already_terminal_task(tmp_path):
|
||||
trace_id = "wd-cancelled"
|
||||
trace_dir = tmp_path / trace_id
|
||||
_write_status(str(trace_dir), trace_id, "cancelled")
|
||||
|
||||
with _patch_trace_dir(tmp_path), patch.object(spawn_tool, "_kill_process_tree") as kill:
|
||||
assert spawn_tool._watch_spawn(
|
||||
trace_id, pid=999999, queue_ticket="t", conversation_id=None,
|
||||
deadline_seconds=0.2, poll_interval=0.05,
|
||||
)
|
||||
kill.assert_not_called()
|
||||
|
||||
|
||||
def test_watchdog_kills_and_fails_hung_runner(tmp_path):
|
||||
trace_id = "wd-hung"
|
||||
trace_dir = tmp_path / trace_id
|
||||
_write_status(str(trace_dir), trace_id, "running")
|
||||
|
||||
with (
|
||||
_patch_trace_dir(tmp_path),
|
||||
patch.object(spawn_tool, "_kill_process_tree") as kill,
|
||||
patch.object(spawn_tool.DeviceExecutionLock, "cancel_reservation") as cancel,
|
||||
patch.object(spawn_tool, "notify") as notifier,
|
||||
):
|
||||
assert not spawn_tool._watch_spawn(
|
||||
trace_id, pid=424242, queue_ticket="ticket-1", conversation_id="conv-1",
|
||||
deadline_seconds=0.2, poll_interval=0.05,
|
||||
)
|
||||
|
||||
kill.assert_called_once_with(424242)
|
||||
cancel.assert_called_once_with("ticket-1")
|
||||
notifier.assert_called_once()
|
||||
|
||||
with open(trace_dir / "status.json", encoding="utf-8") as f:
|
||||
status = json.load(f)
|
||||
assert status["status"] == "failed"
|
||||
assert "no logs" in status.get("error", "")
|
||||
|
||||
|
||||
def test_watchdog_tolerates_logs_appearing_at_deadline(tmp_path):
|
||||
"""Logs that appear between the last poll and the deadline must not cause a kill."""
|
||||
trace_id = "wd-late-boot"
|
||||
trace_dir = tmp_path / trace_id
|
||||
_write_status(str(trace_dir), trace_id, "running")
|
||||
|
||||
real_read_status = spawn_tool.trace_store.read_status
|
||||
calls = {"n": 0}
|
||||
|
||||
def read_status_and_create_log(tid):
|
||||
calls["n"] += 1
|
||||
if calls["n"] == 1:
|
||||
(trace_dir / "stderr.log").write_text("late boot", encoding="utf-8")
|
||||
return real_read_status(tid)
|
||||
|
||||
with (
|
||||
_patch_trace_dir(tmp_path),
|
||||
patch.object(spawn_tool.trace_store, "read_status", side_effect=read_status_and_create_log),
|
||||
patch.object(spawn_tool, "_kill_process_tree") as kill,
|
||||
):
|
||||
assert spawn_tool._watch_spawn(
|
||||
trace_id, pid=999999, queue_ticket="t", conversation_id=None,
|
||||
deadline_seconds=0.15, poll_interval=0.2,
|
||||
)
|
||||
kill.assert_not_called()
|
||||
@@ -0,0 +1,133 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Regression tests for mobile_manage_task's state reconciliation.
|
||||
|
||||
Guards the invariants restored after the reconcile rewrite:
|
||||
- a DB terminal verdict never overrides a status.json that already reached its
|
||||
own terminal state (a completed task must keep its result),
|
||||
- except to correct a previously liveness-inferred failure,
|
||||
- and a task without a recorded pid (daemon dispatch) is never inferred dead.
|
||||
"""
|
||||
|
||||
import sqlite3
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
import mcp_server.tools.task_manager as task_manager
|
||||
from mcp_server.tools.task_manager import _LIVENESS_FAILURE_ERROR, _reconcile_task_state
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def no_persistence(monkeypatch):
|
||||
"""Keep reconciliation in-memory: no DB, no status writes, no lock queries."""
|
||||
monkeypatch.setattr(task_manager, "_find_data_engine_db", lambda: None)
|
||||
monkeypatch.setattr(task_manager, "_session_tracked_by_lock", lambda _tid: False)
|
||||
monkeypatch.setattr(task_manager.trace_store, "write_status", lambda *_a, **_k: None)
|
||||
monkeypatch.setattr(
|
||||
task_manager.trace_store, "update_trace_status", lambda *_a, **_k: None
|
||||
)
|
||||
monkeypatch.setattr(task_manager, "notify", lambda *_a, **_k: None)
|
||||
|
||||
|
||||
def _db_with_session(tmp_path, trace_id: str, status: str, pid: int | None = 4242):
|
||||
db_path = tmp_path / "artemis.db"
|
||||
conn = sqlite3.connect(db_path)
|
||||
conn.execute(
|
||||
"CREATE TABLE sessions (session_id TEXT, status TEXT, pid INTEGER, start_time REAL)"
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO sessions VALUES (?, ?, ?, ?)", (trace_id, status, pid, time.time())
|
||||
)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
return str(db_path)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def use_db(tmp_path, monkeypatch, no_persistence):
|
||||
def _install(trace_id: str, status: str, pid: int | None = 4242):
|
||||
db_path = _db_with_session(tmp_path, trace_id, status, pid)
|
||||
monkeypatch.setattr(task_manager, "_find_data_engine_db", lambda: db_path)
|
||||
|
||||
return _install
|
||||
|
||||
|
||||
def test_db_failed_never_overrides_completed_status(use_db):
|
||||
use_db("t1", "failed")
|
||||
status_data = {"status": "completed", "result": "the answer", "pid": 111}
|
||||
|
||||
current_status, _pid, is_alive = _reconcile_task_state("t1", status_data)
|
||||
|
||||
assert current_status == "completed"
|
||||
assert status_data["status"] == "completed"
|
||||
assert status_data["result"] == "the answer"
|
||||
assert is_alive is False
|
||||
|
||||
|
||||
def test_db_verdict_corrects_liveness_inferred_failure(use_db):
|
||||
use_db("t2", "completed")
|
||||
status_data = {"status": "failed", "error": _LIVENESS_FAILURE_ERROR, "pid": 111}
|
||||
|
||||
current_status, _pid, _is_alive = _reconcile_task_state("t2", status_data)
|
||||
|
||||
assert current_status == "completed"
|
||||
assert status_data["error"] is None
|
||||
|
||||
|
||||
def test_db_never_overrides_user_cancellation(use_db):
|
||||
use_db("t3", "completed")
|
||||
status_data = {"status": "cancelled", "pid": 111}
|
||||
|
||||
current_status, _pid, _is_alive = _reconcile_task_state("t3", status_data)
|
||||
|
||||
assert current_status == "cancelled"
|
||||
|
||||
|
||||
def test_pidless_running_task_is_assumed_alive(no_persistence):
|
||||
# Daemon dispatch writes no pid into status.json; liveness must not be
|
||||
# inferred even long past the startup grace window.
|
||||
status_data = {"status": "running", "start_time": time.time() - 3600}
|
||||
|
||||
current_status, _pid, is_alive = _reconcile_task_state("t4", status_data)
|
||||
|
||||
assert current_status == "running"
|
||||
assert is_alive is True
|
||||
assert status_data.get("error") is None
|
||||
|
||||
|
||||
def test_dead_pid_past_grace_is_marked_failed(no_persistence, monkeypatch):
|
||||
monkeypatch.setattr(task_manager, "_pid_alive", lambda _pid: False)
|
||||
status_data = {
|
||||
"status": "running",
|
||||
"pid": 99999,
|
||||
"start_time": time.time() - 3600,
|
||||
}
|
||||
|
||||
current_status, _pid, is_alive = _reconcile_task_state("t5", status_data)
|
||||
|
||||
assert current_status == "failed"
|
||||
assert status_data["error"] == _LIVENESS_FAILURE_ERROR
|
||||
assert is_alive is False
|
||||
|
||||
|
||||
def test_dead_pid_within_grace_is_assumed_alive(no_persistence, monkeypatch):
|
||||
monkeypatch.setattr(task_manager, "_pid_alive", lambda _pid: False)
|
||||
status_data = {"status": "running", "pid": 99999, "start_time": time.time()}
|
||||
|
||||
current_status, _pid, is_alive = _reconcile_task_state("t6", status_data)
|
||||
|
||||
assert current_status == "running"
|
||||
assert is_alive is True
|
||||
@@ -14,6 +14,7 @@
|
||||
|
||||
"""Unit tests for MCP trace store."""
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import uuid
|
||||
@@ -80,6 +81,6 @@ def test_read_nonexistent_status(temp_trace_env):
|
||||
def test_trace_paths(temp_trace_env):
|
||||
trace_id = "test-paths-id"
|
||||
trace_dir = trace_store.get_trace_dir(trace_id)
|
||||
assert trace_store.get_trace_notes_dir(trace_id) == f"{trace_dir}/notes"
|
||||
assert trace_store.get_trace_stdout_log_path(trace_id) == f"{trace_dir}/stdout.log"
|
||||
assert trace_store.get_trace_stderr_log_path(trace_id) == f"{trace_dir}/stderr.log"
|
||||
assert trace_store.get_trace_notes_dir(trace_id) == os.path.join(trace_dir, "notes")
|
||||
assert trace_store.get_trace_stdout_log_path(trace_id) == os.path.join(trace_dir, "stdout.log")
|
||||
assert trace_store.get_trace_stderr_log_path(trace_id) == os.path.join(trace_dir, "stderr.log")
|
||||
+66
-1
@@ -222,7 +222,9 @@ def test_cli_mcp_install_all(tmp_path, monkeypatch):
|
||||
assert (tmp_path / ".gemini" / "rules" / "artemis.md").exists()
|
||||
assert (tmp_path / ".cursorrules").exists()
|
||||
assert (tmp_path / ".cursor" / "rules" / "artemis.mdc").exists()
|
||||
assert (tmp_path / ".claude" / "CLAUDE.md").exists()
|
||||
# Claude Code loads both CLAUDE.md and rules/*.md, so rules are installed
|
||||
# only as the standalone rule file to avoid duplicated context.
|
||||
assert not (tmp_path / ".claude" / "CLAUDE.md").exists()
|
||||
assert (tmp_path / ".claude" / "rules" / "artemis.md").exists()
|
||||
assert (tmp_path / ".codeium" / "windsurf" / "memories" / "global_rules.md").exists()
|
||||
assert (tmp_path / ".codeium" / "windsurf" / "rules" / "artemis.md").exists()
|
||||
@@ -236,6 +238,69 @@ def test_cli_mcp_install_all(tmp_path, monkeypatch):
|
||||
assert (tmp_path / ".codex" / "AGENTS.md").exists()
|
||||
|
||||
|
||||
def test_cli_mcp_install_claude_migrates_legacy_claude_md(tmp_path, monkeypatch):
|
||||
"""Verify the claude target removes a previously injected CLAUDE.md block while preserving user content."""
|
||||
monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path)
|
||||
monkeypatch.setenv("APPDATA", str(tmp_path / "AppData" / "Roaming"))
|
||||
monkeypatch.setattr("mcp_server.utils.env_utils.get_project_root", lambda: str(tmp_path))
|
||||
|
||||
claude_dir = tmp_path / ".claude"
|
||||
claude_dir.mkdir(parents=True)
|
||||
claude_md = claude_dir / "CLAUDE.md"
|
||||
claude_md.write_text(
|
||||
"# My own instructions\n\n"
|
||||
"<!-- BEGIN ARTEMIS MOBILE TESTING RULES -->\nold injected rules\n"
|
||||
"<!-- END ARTEMIS MOBILE TESTING RULES -->\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
result = runner.invoke(app, ["mcp", "--install", "claude"])
|
||||
assert result.exit_code == 0
|
||||
|
||||
remaining = claude_md.read_text(encoding="utf-8")
|
||||
assert "My own instructions" in remaining
|
||||
assert "ARTEMIS MOBILE TESTING RULES" not in remaining
|
||||
rule_file = claude_dir / "rules" / "artemis.md"
|
||||
assert rule_file.exists()
|
||||
assert "Mobile Testing Mindset" in rule_file.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_cli_mcp_install_claude_deletes_block_only_claude_md(tmp_path, monkeypatch):
|
||||
"""Verify a CLAUDE.md consisting solely of the injected block is deleted outright."""
|
||||
monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path)
|
||||
monkeypatch.setenv("APPDATA", str(tmp_path / "AppData" / "Roaming"))
|
||||
monkeypatch.setattr("mcp_server.utils.env_utils.get_project_root", lambda: str(tmp_path))
|
||||
|
||||
claude_dir = tmp_path / ".claude"
|
||||
claude_dir.mkdir(parents=True)
|
||||
claude_md = claude_dir / "CLAUDE.md"
|
||||
claude_md.write_text(
|
||||
"<!-- BEGIN ARTEMIS MOBILE TESTING RULES -->\nold injected rules\n"
|
||||
"<!-- END ARTEMIS MOBILE TESTING RULES -->\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
result = runner.invoke(app, ["mcp", "--install", "claude"])
|
||||
assert result.exit_code == 0
|
||||
assert not claude_md.exists()
|
||||
assert (claude_dir / "rules" / "artemis.md").exists()
|
||||
|
||||
|
||||
def test_cli_mcp_install_claude_refuses_unparseable_claude_json(tmp_path, monkeypatch):
|
||||
"""Verify an unparseable ~/.claude.json is left untouched instead of being rewritten."""
|
||||
monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path)
|
||||
monkeypatch.setenv("APPDATA", str(tmp_path / "AppData" / "Roaming"))
|
||||
monkeypatch.setattr("mcp_server.utils.env_utils.get_project_root", lambda: str(tmp_path))
|
||||
|
||||
corrupt = "{ this is not json at all"
|
||||
claude_json = tmp_path / ".claude.json"
|
||||
claude_json.write_text(corrupt, encoding="utf-8")
|
||||
|
||||
result = runner.invoke(app, ["mcp", "--install", "claude"])
|
||||
assert result.exit_code == 0
|
||||
assert claude_json.read_text(encoding="utf-8") == corrupt
|
||||
|
||||
|
||||
def test_cli_mcp_install_codex_preserves_config_and_is_idempotent(tmp_path, monkeypatch):
|
||||
"""Verify Codex TOML merging preserves other servers and updates only Artemis."""
|
||||
monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path)
|
||||
|
||||
Reference in new issue
Block a user