feat: extract in-process MCP action layer, concurrent task dispatch, and spawn/reconcile hardening

- Extract device action execution into artemis/mcp action layer:
  typed ActionResult, capability manifest with startup validation,
  AdbActuator as the single physical implementation, in-memory
  ActionSession (replaces stdio subprocess MCP in validator), and
  McpActionExecutor preserving the legacy executor contract.
  adb_server keeps its external stdio/SSE surface unchanged and now
  reuses the actuator implementation; import-time side effects moved
  into configure_stdio_mode().
- Capability-gated prompt assembly (prompt_assembly.py): tool enums and
  teaching segments render per available backend tools; full-set output
  is byte-identical to the previous static text (locked by tests).
- Admin console task queue: serial worker replaced by a per-device
  concurrent dispatcher (ARTEMIS_CONCURRENCY_MODE=global restores
  serial semantics), per-session cancellation, strict device binding on
  enqueue, unified stdio piping with tee to trace stdout.log, and
  shutdown cleanup of all active runs. Fix _pid_exists exception order
  so PermissionError is treated as alive.
- mcp_server reliability: spawn watchdog kills runners stuck before
  interpreter startup and releases device reservations; task state
  reconcile uses psutil liveness, prefers runner-reported pids, and
  adds a 45s startup grace to stop false failure reports on Windows;
  conversation_id is now optional end-to-end; explicit device_serial is
  validated at submission.
- CLI install now writes ~/.claude/rules/artemis.md for Claude Code and
  migrates/removes the legacy CLAUDE.md injection block; ~/.claude.json
  is never rewritten when it fails to parse.
- Tests: Windows-portable stdio lifecycle and path assertions; new
  suites for action layer, prompt assembly, spawn watchdog, and task
  reconcile (895 passed, 3 skipped).
This commit is contained in:
weilinliu55 committed 2026-08-27 18:25:27 -07:00
1 parent 60312abbb0
commit cbdca1497c
52 files changed
+5735 -1649

No files matched your search

@@ -667,6 +667,10 @@ async def test_queue_worker_notifies_conversation():
patch("apps.admin_console.services.task_queue_service.media_service"),
patch("mcp_server.notifiers.notify") as mock_notify,
patch("apps.admin_console.services.task_queue_service.session_repo") as mock_repo,
patch(
"artemis.runtime.device_pool.device_pool.list_devices_async",
new=AsyncMock(return_value=[]),
),
):
mock_repo.get_running_session_id.return_value = None
@@ -849,6 +853,11 @@ async def test_enqueue_tasks_debounces_rapid_identical_submissions():
with (
patch("apps.admin_console.services.task_queue_service.session_repo"),
patch("apps.admin_console.services.task_queue_service.DeviceExecutionLock.reserve", return_value="ticket-456"),
patch(
"artemis.runtime.device_pool.device_pool.list_devices_async",
new=AsyncMock(return_value=[]),
),
patch.object(TaskQueueService, "ensure_worker_running"),
):
res1 = await task_queue_service.enqueue_tasks(["Rapid duplicate goal"], device_serial="dev-1")
assert len(state.queue_items) == 1
@@ -304,9 +304,11 @@ async def test_input_text_happy_path(mock_context, mock_controller):
ctx=mock_context,
)
assert "Executed typing 'hello'." in outcome
mock_controller.tap_at.assert_called_once_with(540, 1200)
# Focus goes through ensure_focus_at_coords (keyword-style tap), and typing
# never re-clears: the clear already happened via erase_text.
mock_controller.tap_at.assert_called_once_with(x=540, y=1200)
mock_controller.erase_text.assert_called_once()
mock_controller.type_text.assert_called_once_with("hello")
mock_controller.type_text.assert_called_once_with("hello", clear_existing=False)
@pytest.mark.asyncio
@@ -396,8 +398,10 @@ async def test_press_key_input_validation(mock_context, mock_controller):
controller=mock_controller,
ctx=mock_context,
)
# The current execution doesn't raise error on unknown keys, just logs error
assert "Error executing key press 'INVALID_KEY'." in outcome
# Unknown keycodes are forwarded verbatim to the driver (historical adb_server
# behavior); the mock driver accepts them, so the press reports success.
assert "Executed key press 'INVALID_KEY'." in outcome
mock_controller.press_key.assert_called_once_with("INVALID_KEY")
# ==========================================
@@ -409,8 +413,10 @@ async def test_press_key_input_validation(mock_context, mock_controller):
async def test_manage_app_happy_path(mock_context, mock_controller):
state = DummyState()
with (
patch("artemis.agents.validator.tool_declarations.find_package") as mock_find_package,
patch("artemis.agents.validator.tool_declarations.launch_app_with_retries") as mock_launch,
# The device-call bodies moved into the actuator layer, which imports these
# lazily from their source modules -- patch them at the source.
patch("artemis.tools.mobile.launch_app.find_package") as mock_find_package,
patch("artemis.utils.app_launch_utils.launch_app_with_retries") as mock_launch,
):
mock_find_package.return_value = "com.google.android.youtube"
mock_launch.return_value = (True, "")
@@ -430,7 +436,7 @@ async def test_manage_app_happy_path(mock_context, mock_controller):
@pytest.mark.asyncio
async def test_manage_app_input_validation(mock_context, mock_controller):
state = DummyState()
with patch("artemis.agents.validator.tool_declarations.find_package") as mock_find_package:
with patch("artemis.tools.mobile.launch_app.find_package") as mock_find_package:
mock_find_package.return_value = None
outcome, _, _, _ = await _exec_manage_app(
+233
View File
@@ -0,0 +1,233 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Prompt assembly tests: full-set output is byte-stable, absent tools leave no trace."""
from pathlib import Path
import re
from jinja2 import Template
import pytest
from artemis.agents.operator.prompt_builder import load_operator_prompts
from artemis.agents.operator.prompts import (
OPERATOR_PROMPT_TOOLSET,
apply_operator_prompt_contract,
)
from artemis.agents.prompt_assembly import gate_segment, render_tool_enum
from artemis.agents.validator.failure_analyzer import (
FailureAnalysisStrategy,
FailureAnalyzer,
PixelTargetDisappearedStrategy,
TargetDisappearedStrategy,
)
from artemis.agents.validator.validator import ValidationErrorCategory
REPO_ROOT = Path(__file__).resolve().parents[3]
OPERATOR_DEVICE_TOOLS = (
"click",
"input_text",
"swipe",
"press_key",
"manage_app",
"wait_for_delay",
"long_press",
)
def _tool_refs(text: str, tool: str) -> list[str]:
"""Finds real tool references (backticked or call form), ignoring prose."""
return re.findall(rf"(`{tool}`|\b{tool}\()", text)
# --- render_tool_enum ----------------------------------------------------------------
def test_enum_plain_list():
out = render_tool_enum(("a", "b", "c"), {"a", "b", "c"})
assert out == "`a`, `b`, `c`"
def test_enum_oxford_or_and():
tools = ("a", "b", "c")
assert render_tool_enum(tools, set(tools), final_sep="or") == "`a`, `b`, or `c`"
assert render_tool_enum(tools, set(tools), final_sep="and") == "`a`, `b`, and `c`"
def test_enum_two_items_no_oxford_comma():
assert render_tool_enum(("a", "b"), {"a", "b"}, final_sep="or") == "`a` or `b`"
def test_enum_single_and_empty():
assert render_tool_enum(("a", "b"), {"a"}) == "`a`"
assert render_tool_enum(("a", "b"), set()) == ""
def test_gate_segment():
assert gate_segment("teach", {"swipe"}, "swipe") == "teach"
assert gate_segment("teach", set(), "swipe") == ""
assert gate_segment("teach", {"a"}, "a", "b") == ""
# --- Operator prompt assembly --------------------------------------------------------
def test_full_toolset_is_the_default():
"""``available_tools=None`` must equal the explicit full set, byte for byte."""
for template in load_operator_prompts().values():
assert apply_operator_prompt_contract(template) == apply_operator_prompt_contract(
template, available_tools=OPERATOR_PROMPT_TOOLSET
)
@pytest.mark.parametrize("tool", OPERATOR_DEVICE_TOOLS + ("video_analyzer",))
def test_removed_tool_leaves_no_reference(tool):
"""The executable definition of 'an absent tool costs the model nothing'."""
for name, template in load_operator_prompts().items():
out = apply_operator_prompt_contract(
template, available_tools=OPERATOR_PROMPT_TOOLSET - {tool}
)
assert not _tool_refs(out, tool), f"'{tool}' still referenced in {name}"
def test_removed_swipe_takes_its_teaching_along():
"""Gating removes the drag/slider teaching, not just the backticked name."""
template = load_operator_prompts()["main_template"]
out = apply_operator_prompt_contract(
template, available_tools=OPERATOR_PROMPT_TOOLSET - {"swipe"}
)
assert "swipe/drag" not in out
assert "sliders/SeekBars" not in out
# The generic gesture-scope sentences survive.
assert "Action Scope & Interaction Precision" in out
def test_removed_wait_for_delay_keeps_interval_line_coherent():
"""Only the wait_for_delay parenthetical goes; the SOP grammar line stays."""
template = load_operator_prompts()["main_template"]
out = apply_operator_prompt_contract(
template, available_tools=OPERATOR_PROMPT_TOOLSET - {"wait_for_delay"}
)
assert "`Interval: <Interval>`: Declared polling cadence." in out
assert "Loading & Transitions" not in out
def test_reduced_enums_stay_well_formed():
template = load_operator_prompts()["main_template"]
out = apply_operator_prompt_contract(
template,
available_tools=OPERATOR_PROMPT_TOOLSET - {"manage_app", "wait_for_delay"},
)
assert (
"Physical device actions (`click`, `input_text`, `swipe`, `press_key`,"
" `long_press`)" in out
)
assert (
"Turn-Ending Action (`click`, `swipe`, `input_text`, `long_press`, or"
" `press_key`)" in out
)
def test_full_set_enum_slots_render_verbatim():
"""With every tool present the historical enumeration wording is reproduced."""
template = load_operator_prompts()["main_template"]
out = apply_operator_prompt_contract(template)
assert (
"Physical device actions (`click`, `input_text`, `swipe`, `press_key`,"
" `manage_app`, `wait_for_delay`, `long_press`)" in out
)
assert (
"Turn-Ending Action (`click`, `swipe`, `input_text`, `long_press`,"
" `press_key`, `manage_app`, or `wait_for_delay`)" in out
)
assert (
"(`ask_explorer`, `ask_diagnoser`, `video_analyzer`, `run_adb_command`,"
" `manage_task`, `analyze_task_output`, `read_note`, `list_notes`, and"
" `save_note`)" in out
)
# --- Flash prompt assembly -----------------------------------------------------------
def _render_flash(available_tools: frozenset[str]) -> str:
template_text = (REPO_ROOT / "artemis/agents/flash/flash_runner.md").read_text(
encoding="utf-8"
)
return Template(template_text).render(goal="test goal", available_tools=available_tools)
FLASH_FULL = frozenset({"manage_app", "wait_for_delay", "click_sequence", "click", "swipe"})
def test_flash_full_set_keeps_both_segments():
out = _render_flash(FLASH_FULL)
assert "**App Launching**" in out
assert "**Timed Waiting & Transitions**" in out
assert "{%" not in out
def test_flash_without_manage_app_drops_app_launching():
out = _render_flash(FLASH_FULL - {"manage_app"})
assert "**App Launching**" not in out
assert not _tool_refs(out, "manage_app")
# The neighbouring numbered rule survives.
assert "**Screenshot Pruning Awareness**" in out
def test_flash_without_wait_for_delay_drops_timed_waiting():
out = _render_flash(FLASH_FULL - {"wait_for_delay"})
assert "**Timed Waiting & Transitions**" not in out
assert not _tool_refs(out, "wait_for_delay")
assert "**State Drift & Progression Tolerance**" in out
# --- FailureAnalyzer strategy gating -------------------------------------------------
def _analyzer() -> FailureAnalyzer:
return FailureAnalyzer.__new__(FailureAnalyzer) # _select_strategy needs no ctx
def test_strategy_selection_unchanged_without_gating():
analyzer = _analyzer()
assert isinstance(
analyzer._select_strategy(ValidationErrorCategory.TARGET_DISAPPEARED),
TargetDisappearedStrategy,
)
assert isinstance(
analyzer._select_strategy(ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED),
PixelTargetDisappearedStrategy,
)
def test_strategy_falls_back_when_required_tool_missing():
"""Strategy + prompt are one unit: no click_sequence, no disappeared-prompts."""
analyzer = _analyzer()
strategy = analyzer._select_strategy(
ValidationErrorCategory.TARGET_DISAPPEARED,
available_tools=frozenset({"click", "swipe"}),
)
assert type(strategy) is FailureAnalysisStrategy
assert strategy.get_prompt_template_name() == "failure_analyzer.md"
def test_strategy_kept_when_required_tool_present():
analyzer = _analyzer()
strategy = analyzer._select_strategy(
ValidationErrorCategory.PIXEL_TARGET_DISAPPEARED,
available_tools=frozenset({"click_sequence"}),
)
assert isinstance(strategy, PixelTargetDisappearedStrategy)
+112 -251
View File
@@ -12,13 +12,25 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""ValidatorNode tests over the in-process unified action session.
The Validator no longer spawns a stdio MCP subprocess; it calls a shared
``ActionSession`` whose device actions return structured ``ActionResult`` objects.
``FakeActionSession`` below stands in for it: ``action_handler`` decides each
action's ``(ok, message)``, ``screenshot`` feeds the polling loop, and ``hierarchy``
feeds the safety-net XML validation.
"""
import json
from unittest.mock import AsyncMock, MagicMock, Mock, patch
from langchain_core.messages import AIMessage
import pytest
from artemis.agents.validator.failure_analyzer import ValidationErrorCategory
from artemis.agents.validator.validator import ValidatorNode
from artemis.context import ArtemisContext
import pytest
from artemis.mcp.action_types import ActionCode, ActionResult
class DummyState:
@@ -38,6 +50,46 @@ class DummyState:
return update
class FakeActionSession:
"""Test double for artemis.mcp.action_session.ActionSession."""
def __init__(self):
self.started = True
self.calls: list[tuple[str, dict]] = []
# (name, args) -> (ok, message)
self.action_handler = lambda name, args: (True, "")
# str | callable | Exception
self.screenshot = "ZHVtbXlfZGF0YQ==" # b64("dummy_data")
self.hierarchy: list | Exception = []
async def call(self, name, args):
self.calls.append((name, args))
ok, msg = self.action_handler(name, args)
return ActionResult(
ok=ok,
code=ActionCode.OK if ok else ActionCode.DEVICE_ERROR,
action=name,
message=msg,
)
async def screenshot_b64(self, timeout=None):
shot = self.screenshot
if isinstance(shot, Exception):
raise shot
return shot() if callable(shot) else shot
async def ui_hierarchy(self, timeout=None):
if isinstance(self.hierarchy, Exception):
raise self.hierarchy
return self.hierarchy
async def aclose(self):
pass
def calls_for(self, name: str) -> list[dict]:
return [args for n, args in self.calls if n == name]
@pytest.fixture
def mock_context(tmp_path):
ctx = Mock(spec=ArtemisContext)
@@ -45,6 +97,9 @@ def mock_context(tmp_path):
ctx.data_engine = Mock()
ctx.data_engine.base_dir = tmp_path
ctx.data_engine.get_relative_time.return_value = 1.0
ctx.device = Mock()
ctx.device.device_width = 1080
ctx.device.device_height = 2400
return ctx
@@ -57,31 +112,18 @@ def temp_screenshot(tmp_path):
@pytest.fixture
def mock_mcp():
with (
patch("mcp.client.stdio.stdio_client") as mock_stdio,
patch("artemis.agents.validator.validator.ClientSession") as mock_session_cls,
):
mock_ctx = AsyncMock()
mock_ctx.__aenter__.return_value = (Mock(), Mock())
mock_stdio.return_value = mock_ctx
fake = FakeActionSession()
mock_session = AsyncMock()
mock_session.initialize = AsyncMock()
mock_session_cls.return_value = mock_session
async def _get_session(ctx, actuator=None):
return fake
yield mock_session
with patch("artemis.agents.validator.validator.get_action_session", _get_session):
yield fake
@pytest.mark.asyncio
async def test_validator_success(mock_mcp, mock_context, temp_screenshot):
"""Test that ValidatorNode executes actions successfully."""
# Mock MCP calls: screenshot, tap, screenshot
mock_mcp.call_tool.side_effect = [
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # pre-screenshot
Mock(content=[Mock(text="Success")]), # tap action
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # post-screenshot
]
decisions = json.dumps([{"action": "tap", "coordinates": [105, 205]}])
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
@@ -95,78 +137,23 @@ async def test_validator_success(mock_mcp, mock_context, temp_screenshot):
assert len(report["execution"]) == 1
assert report["execution"][0]["action"] == "tap"
assert "attempts" not in report["execution"][0]
# The Operator's tap verb was translated to the canonical click tool.
assert len(mock_mcp.calls_for("click")) == 1
@pytest.mark.asyncio
async def test_failed_mcp_handshake_exits_contexts_and_disables_child_awake_policy():
"""A handshake failure must not leave AnyIO cancellation scopes behind."""
events = []
class FakeClientContext:
async def __aenter__(self):
events.append("client_enter")
return Mock(), Mock()
async def __aexit__(self, exc_type, exc, traceback):
events.append("client_exit")
class FakeSession:
async def __aenter__(self):
events.append("session_enter")
return self
async def __aexit__(self, exc_type, exc, traceback):
events.append("session_exit")
async def initialize(self):
raise TimeoutError("handshake timed out")
ctx = Mock(spec=ArtemisContext)
ctx.mcp_client_ctx = None
ctx.mcp_session = None
fake_client_ctx = FakeClientContext()
fake_session = FakeSession()
captured_parameters = []
def fake_stdio_client(parameters):
captured_parameters.append(parameters)
return fake_client_ctx
node = ValidatorNode(ctx)
with (
patch("artemis.agents.validator.validator.stdio_client", fake_stdio_client),
patch("artemis.agents.validator.validator.ClientSession", return_value=fake_session),
patch("artemis.agents.validator.validator.AdbClient") as adb_client,
patch.object(node, "_kill_mcp_server_instantly") as emergency_kill,
):
adb_client.return_value.device_list.return_value = []
result = await node._get_mcp_session()
assert result is None
assert events == ["client_enter", "session_enter", "session_exit", "client_exit"]
assert ctx.mcp_session is None
assert ctx.mcp_client_ctx is None
assert captured_parameters[0].env["ARTEMIS_KEEP_DEVICE_AWAKE"] == "false"
emergency_kill.assert_not_called()
# NOTE: upstream's test_failed_mcp_handshake_exits_contexts_and_disables_child_awake_policy
# guarded the stdio-subprocess handshake teardown, machinery this migration removed
# entirely. Its hazard class (leaked AnyIO cancel scopes) is now pinned by
# tests/unit/mcp/test_action_session.py against the in-process ActionSession.
@pytest.mark.asyncio
async def test_validator_failure_analysis(mock_mcp, mock_context, temp_screenshot):
"""Test that ValidatorNode triggers failure analysis on error and handles
cannot_fix via tool.
"""
tap_count = 0
def mock_call_tool(name, args):
nonlocal tap_count
if name == "take_screenshot":
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
if name == "tap":
tap_count += 1
return Mock(content=[Mock(text="Error: Element not found")])
return Mock(content=[Mock(text="Success")])
mock_mcp.call_tool.side_effect = mock_call_tool
mock_mcp.action_handler = lambda name, args: (
(False, "Error: Element not found") if name == "click" else (True, "")
)
decisions = json.dumps([{"action": "tap", "coordinates": [105, 205]}])
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
@@ -203,18 +190,15 @@ async def test_validator_repair_success(mock_mcp, mock_context, temp_screenshot)
"""
tap_count = 0
def mock_call_tool(name, args):
def handler(name, args):
nonlocal tap_count
if name == "take_screenshot":
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
if name == "tap":
if name == "click":
tap_count += 1
if tap_count <= 2: # First tap fails
return Mock(content=[Mock(text="Error: Element not found")])
return Mock(content=[Mock(text="Success")])
return Mock(content=[Mock(text="Success")])
if tap_count <= 2: # First action fails on both validator attempts
return False, "Error: Element not found"
return True, ""
mock_mcp.call_tool.side_effect = mock_call_tool
mock_mcp.action_handler = handler
# Two actions planned initially
decisions = json.dumps(
@@ -254,12 +238,6 @@ async def test_validator_repair_success(mock_mcp, mock_context, temp_screenshot)
@pytest.mark.asyncio
async def test_validator_wait_for_delay(mock_mcp, mock_context, temp_screenshot):
"""Test that wait_for_delay executes successfully."""
def mock_call(name, args):
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
mock_mcp.call_tool.side_effect = mock_call
decisions = json.dumps([{"action": "wait_for_delay", "time_in_ms": 10}])
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
@@ -272,19 +250,13 @@ async def test_validator_wait_for_delay(mock_mcp, mock_context, temp_screenshot)
assert "execution" in report
assert len(report["execution"]) == 1
assert report["execution"][0]["action"] == "wait_for_delay"
# wait_for_delay is a pure client-side sleep, never a session call.
assert mock_mcp.calls_for("wait_for_delay") == []
@pytest.mark.asyncio
async def test_validator_focus_and_clear_text_no_ui_change(mock_mcp, mock_context, temp_screenshot):
"""Test that focus_and_clear_text executes successfully even if no UI change is detected."""
def mock_call(name, args):
if name == "focus_and_clear_text":
return Mock(content=[Mock(text="Success")])
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
mock_mcp.call_tool.side_effect = mock_call
decisions = json.dumps([{"action": "focus_and_clear_text", "coordinates": [400, 500]}])
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
@@ -297,6 +269,7 @@ async def test_validator_focus_and_clear_text_no_ui_change(mock_mcp, mock_contex
assert "execution" in report
assert len(report["execution"]) == 1
assert report["execution"][0]["action"] == "focus_and_clear_text"
assert len(mock_mcp.calls_for("focus_and_clear_text")) == 1
@pytest.mark.asyncio
@@ -304,16 +277,6 @@ async def test_validator_silent_failure_treated_as_success(mock_mcp, mock_contex
"""Test that silent failure (exec succeeds but no UI change) is treated as
success and does not trigger FailureAnalyzer.
"""
def mock_call_tool(name, args):
if name == "take_screenshot":
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
if name == "tap":
return Mock(content=[Mock(text="Success")])
return Mock(content=[Mock(text="Success")])
mock_mcp.call_tool.side_effect = mock_call_tool
decisions = json.dumps([{"action": "tap", "coordinates": [105, 205]}])
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
@@ -344,13 +307,7 @@ async def test_validator_records_skipped_actions_on_cannot_fix(
mock_mcp, mock_context, temp_screenshot
):
"""Test that ValidatorNode marks unexecuted actions as Skipped when repair fails."""
def mock_call_tool(name, args):
if name == "take_screenshot":
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
return Mock(content=[Mock(text="Error: Connection lost")])
mock_mcp.call_tool.side_effect = mock_call_tool
mock_mcp.action_handler = lambda name, args: (False, "Error: Connection lost")
# Two actions planned
decisions = json.dumps(
@@ -399,7 +356,7 @@ async def test_validator_pre_execution_validation_and_repair(
"""Test that pre-execution validation fails when element changes, and is
repaired by FailureAnalyzer.
"""
mock_live_elements = [
mock_mcp.hierarchy = [
{
"text": "Sign Up",
"bounds": "[100,200][300,300]",
@@ -407,17 +364,6 @@ async def test_validator_pre_execution_validation_and_repair(
}
]
def mock_call_tool(name, args):
if name == "take_screenshot":
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
if name == "get_ui_hierarchy":
return Mock(content=[Mock(text=json.dumps(mock_live_elements))])
if name == "tap":
return Mock(content=[Mock(text="Success")])
return Mock(content=[Mock(text="Success")])
mock_mcp.call_tool.side_effect = mock_call_tool
decisions = json.dumps(
[
{
@@ -477,7 +423,7 @@ async def test_validator_pre_execution_validation_self_healing(
mock_mcp, mock_context, temp_screenshot
):
"""Test that pre-execution validation succeeds and self-heals when element shifts slightly."""
mock_live_elements = [
mock_mcp.hierarchy = [
{
"text": "Login",
"bounds": "[100,220][300,320]",
@@ -485,21 +431,6 @@ async def test_validator_pre_execution_validation_self_healing(
}
]
tapped_coordinates = None
def mock_call_tool(name, args):
nonlocal tapped_coordinates
if name == "take_screenshot":
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
if name == "get_ui_hierarchy":
return Mock(content=[Mock(text=json.dumps(mock_live_elements))])
if name == "tap":
tapped_coordinates = args.get("coordinates")
return Mock(content=[Mock(text="Success")])
return Mock(content=[Mock(text="Success")])
mock_mcp.call_tool.side_effect = mock_call_tool
decisions = json.dumps(
[
{
@@ -518,8 +449,12 @@ async def test_validator_pre_execution_validation_self_healing(
with patch("artemis.utils.image_diff.check_ui_change", return_value=True):
await node(state)
# Center of [100, 220][300, 320] is [200, 270]
assert tapped_coordinates == [200, 270]
# Center of [100, 220][300, 320] is pixel [200, 270]; the healed coordinates
# travel to the canonical click tool normalized to 0-1000 on a 1080x2400 screen:
# round(200 * 1000 / 1080) = 185, round(270 * 1000 / 2400) = 112.
clicks = mock_mcp.calls_for("click")
assert len(clicks) == 1
assert clicks[0]["target"] == [185, 112]
@pytest.mark.asyncio
@@ -531,7 +466,7 @@ async def test_validator_pre_execution_validation_anonymous_occupant(
"""
# The live screen contains an anonymous clickable view covering
# target coordinates [200, 250]
mock_live_elements = [
mock_mcp.hierarchy = [
{
"text": "",
"bounds": "[150,200][250,300]",
@@ -540,15 +475,6 @@ async def test_validator_pre_execution_validation_anonymous_occupant(
}
]
def mock_call_tool(name, args):
if name == "take_screenshot":
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
if name == "get_ui_hierarchy":
return Mock(content=[Mock(text=json.dumps(mock_live_elements))])
return Mock(content=[Mock(text="Success")])
mock_mcp.call_tool.side_effect = mock_call_tool
decisions = json.dumps(
[
{
@@ -596,12 +522,6 @@ async def test_validator_pre_execution_validation_anonymous_occupant(
@pytest.mark.asyncio
async def test_validator_pixel_validation_success(mock_mcp, mock_context, temp_screenshot):
"""Test that the pixel safety net succeeds when Gemini reports target is present."""
mock_mcp.call_tool.side_effect = [
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # live screenshot for pixel check
Mock(content=[Mock(text="Success")]), # tap action
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # post-screenshot
]
decisions = json.dumps([{"action": "tap", "coordinates": [100, 200]}])
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
@@ -645,14 +565,6 @@ async def test_validator_pixel_validation_failure(mock_mcp, mock_context, temp_s
"""Test that the pixel safety net fails and triggers FailureAnalyzer
when Gemini reports target is missing.
"""
def mock_call_tool(name, args):
if name == "take_screenshot":
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
return Mock(content=[Mock(text="Success")])
mock_mcp.call_tool.side_effect = mock_call_tool
decisions = json.dumps([{"action": "tap", "coordinates": [100, 200]}])
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
@@ -697,23 +609,15 @@ async def test_validator_pixel_validation_failure(mock_mcp, mock_context, temp_s
@pytest.mark.asyncio
@patch("artemis.agents.validator.validator.find_package")
@patch("artemis.utils.app_launch_utils.launch_app_with_retries")
async def test_validator_launch_app_passes_use_fallback_mcp(
mock_launch_app, mock_find_package, mock_mcp, mock_context, temp_screenshot
async def test_validator_launch_app_routes_through_manage_app(
mock_mcp, mock_context, temp_screenshot
):
"""Test that ValidatorNode calls find_package with use_fallback=False
for launch_app action in MCP mode.
"""Test that a launch_app action item routes to the canonical manage_app tool.
Package resolution (find_package with use_fallback=False) now happens inside the
actuator's manage_app implementation and is covered by the failure-analyzer tool
tests; the Validator's contract is the translation and single dispatch.
"""
mock_find_package.return_value = "com.example.app"
mock_launch_app.return_value = (True, None)
# Mock MCP calls: screenshot (pre), screenshot (post)
mock_mcp.call_tool.side_effect = [
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # pre-screenshot
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # post-screenshot
]
decisions = json.dumps([{"action": "launch_app", "app_name": "My App"}])
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
@@ -721,27 +625,17 @@ async def test_validator_launch_app_passes_use_fallback_mcp(
node = ValidatorNode(mock_context)
result = await node(state)
mock_find_package.assert_called_once_with(mock_context, "My App", use_fallback=False)
mock_launch_app.assert_called_once_with(mock_context, "com.example.app")
manage_calls = mock_mcp.calls_for("manage_app")
assert manage_calls == [{"action": "launch", "app_name": "My App"}]
assert "last_execution_result" in result
@pytest.mark.asyncio
@patch("artemis.agents.validator.validator.find_package")
@patch("artemis.utils.app_launch_utils.launch_app_with_retries")
async def test_validator_launch_app_failure_no_retry(
mock_launch_app, mock_find_package, mock_mcp, mock_context, temp_screenshot
):
async def test_validator_launch_app_failure_no_retry(mock_mcp, mock_context, temp_screenshot):
"""Test that ValidatorNode does NOT retry launch_app action on failure."""
mock_find_package.return_value = "com.example.app"
mock_launch_app.return_value = (False, "Error: Force close")
def mock_call_tool(name, args):
if name == "take_screenshot":
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
return Mock(content=[Mock(text="Success")])
mock_mcp.call_tool.side_effect = mock_call_tool
mock_mcp.action_handler = lambda name, args: (
(False, "Error: Force close") if name == "manage_app" else (True, "")
)
decisions = json.dumps([{"action": "launch_app", "app_name": "My App"}])
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
@@ -760,13 +654,9 @@ async def test_validator_launch_app_failure_no_retry(
node = ValidatorNode(mock_context)
result = await node(state)
# Verify find_package was called only once
mock_find_package.assert_called_once_with(mock_context, "My App", use_fallback=False)
# Verify launch_app_with_retries was called only once
# (no retry by ValidatorNode itself since it relies on launch_app_with_retries)
mock_launch_app.assert_called_once_with(mock_context, "com.example.app")
# launch_app allows a single attempt -- no validator-level retry.
assert len(mock_mcp.calls_for("manage_app")) == 1
# Verify attempts list has only 1 element
assert "last_execution_result" in result
report = result["last_execution_result"]
assert len(report["execution"]) == 1
@@ -783,7 +673,7 @@ async def test_validator_pre_execution_validation_disappeared_not_shifted(
"""
# Live hierarchy only contains a giant background/container
# containing the text "Sign Up"
mock_live_elements = [
mock_mcp.hierarchy = [
{
"text": "Sign Up",
"bounds": "[0,0][1080,2400]",
@@ -791,15 +681,6 @@ async def test_validator_pre_execution_validation_disappeared_not_shifted(
}
]
def mock_call_tool(name, args):
if name == "take_screenshot":
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
if name == "get_ui_hierarchy":
return Mock(content=[Mock(text=json.dumps(mock_live_elements))])
return Mock(content=[Mock(text="Success")])
mock_mcp.call_tool.side_effect = mock_call_tool
decisions = json.dumps(
[
{
@@ -850,12 +731,6 @@ async def test_validator_pre_execution_validation_ocr_direct_to_pixel(
"""Test that OCR-derived elements bypass XML validation and
directly use pixel-based validation.
"""
mock_mcp.call_tool.side_effect = [
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # live screenshot for pixel check
Mock(content=[Mock(text="Success")]), # tap action
Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")]), # post-screenshot
]
decisions = json.dumps(
[
{
@@ -905,7 +780,7 @@ async def test_validator_pre_execution_xml_failure_not_overridden_when_pixel_byp
"""Test that when XML validation fails AND Pixel validation is bypassed
(PIXEL_BYPASSED), the XML failure is NOT overridden.
"""
mock_live_elements = [
mock_mcp.hierarchy = [
{
"text": "Sign Up",
"bounds": "[100,200][300,300]",
@@ -913,15 +788,6 @@ async def test_validator_pre_execution_xml_failure_not_overridden_when_pixel_byp
}
]
def mock_call_tool(name, args):
if name == "take_screenshot":
return Mock(content=[Mock(text="ZHVtbXlfZGF0YQ==")])
if name == "get_ui_hierarchy":
return Mock(content=[Mock(text=json.dumps(mock_live_elements))])
return Mock(content=[Mock(text="Success")])
mock_mcp.call_tool.side_effect = mock_call_tool
decisions = json.dumps(
[
{
@@ -970,17 +836,13 @@ async def test_validator_pre_execution_xml_failure_not_overridden_when_pixel_byp
async def test_validator_failure_screenshot_mcp_error_handled_safely(
mock_mcp, mock_context, temp_screenshot
):
"""Test that when take_screenshot fails on MCP server and returns an error text,
the ValidatorNode does not crash with base64 decoding error.
"""Test that when take_screenshot fails on the session, the ValidatorNode does
not crash -- the failure screenshot is simply absent.
"""
def mock_call_tool(name, args):
if name == "take_screenshot":
return Mock(content=[Mock(text="Error: ADB connection lost")])
if name == "tap":
return Mock(content=[Mock(text="Error: Target not found")])
return Mock(content=[Mock(text="Success")])
mock_mcp.call_tool.side_effect = mock_call_tool
mock_mcp.screenshot = RuntimeError("Error: ADB connection lost")
mock_mcp.action_handler = lambda name, args: (
(False, "Error: Target not found") if name == "click" else (True, "")
)
decisions = json.dumps([{"action": "tap", "coordinates": [105, 205]}])
state = DummyState(structured_decisions=decisions, latest_screenshot=temp_screenshot)
@@ -1002,4 +864,3 @@ async def test_validator_failure_screenshot_mcp_error_handled_safely(
assert report["status"] == "failed"
assert len(report["execution"]) == 1
assert report["execution"][0]["action"] == "tap"
+3 -1
View File
@@ -55,7 +55,9 @@ def test_darwin_platform_paths():
"""Verify macOS platform paths adhere to Apple Library standards."""
darwin_p = DarwinPlatform()
assert darwin_p.os_type == OSType.MACOS
assert "Library/Application Support" in str(darwin_p.paths.config_dir)
# Compare path components so the assertion holds regardless of the host
# OS path separator used by str().
assert darwin_p.paths.config_dir.parts[-3:] == ("Library", "Application Support", "artemis")
assert darwin_p.process.path_separator == ":"
@@ -392,10 +392,12 @@ async def test_validator_pre_execution_loop_reverted_to_exact_safety_contract(
mock_artemis_ctx.data_engine = MagicMock()
node = ValidatorNode(mock_artemis_ctx)
# The Validator now talks to the in-process unified action session.
session = MagicMock()
session.call_tool = AsyncMock(
return_value=MagicMock(content=[MagicMock(text="ZHVtbXlfZGF0YQ==")])
)
session.started = True
session.screenshot_b64 = AsyncMock(return_value="ZHVtbXlfZGF0YQ==")
session.ui_hierarchy = AsyncMock(return_value=[])
session.call = AsyncMock()
decisions = json.dumps(
[
@@ -421,6 +423,10 @@ async def test_validator_pre_execution_loop_reverted_to_exact_safety_contract(
)
with (
patch(
"artemis.agents.validator.validator.get_action_session",
AsyncMock(return_value=session),
),
patch.object(
ValidatorNode, "_get_initial_screenshot", new_callable=AsyncMock
) as mock_get_screen,
+294
View File
@@ -0,0 +1,294 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Prompt <-> manifest consistency and actuator contract tests.
The scan below re-derives the manifest classification from the prompt sources, so
adding a tool name to a prompt without classifying it in
``artemis/mcp/action_manifest.py`` fails here rather than at runtime.
"""
import json
from pathlib import Path
import re
import pytest
from artemis.core.tool_declaration import ToolDeclaration
from artemis.mcp.action_manifest import (
ActuatorContractError,
AGENT_ROLES,
ALL_KNOWN_TOOLS,
BACKEND_INDEPENDENT_TOOLS,
DEVICE_ACTIONS,
ExtensionTool,
INTERNAL_ACTIONS,
OPTIONAL_ACTIONS,
REQUIRED_ACTIONS,
available_device_actions,
filter_declarations,
validate_actuator,
)
REPO_ROOT = Path(__file__).resolve().parents[3]
# Prompt sources belonging to the three agents this manifest governs (plus the
# Validator's safety net). Other agents (log_analyzer, image_processor, ...) have their
# own tool universes and are deliberately out of scope.
SCOPED_PROMPT_FILES = [
"artemis/agents/flash/flash_runner.md",
"artemis/agents/validator/failure_analyzer.md",
"artemis/agents/validator/target_disappeared_analyzer.md",
"artemis/agents/validator/pixel_target_disappeared_analyzer.md",
"artemis/agents/validator/pixel_safety_net.md",
]
# Backtick-quoted identifiers in scoped prompts that are not tools: parameter names and
# note keys. Anything new landing here should be a conscious decision.
NON_TOOL_IDENTIFIERS = frozenset({"analysis", "task_plan"})
_TOOL_REF = re.compile(r"(?:`([a-z][a-z0-9_]{2,})`|\b([a-z][a-z0-9_]{2,})\()")
def _scoped_prompt_sources() -> dict[str, str]:
"""Collects all in-scope prompt texts, keyed by a short source label."""
sources: dict[str, str] = {}
for rel in SCOPED_PROMPT_FILES:
path = REPO_ROOT / rel
sources[path.name] = path.read_text(encoding="utf-8")
operator_json = REPO_ROOT / "artemis/agents/operator/operator.json"
for key, text in json.loads(operator_json.read_text(encoding="utf-8")).items():
sources[f"operator.json:{key}"] = text
# Only the prompt string constants from prompts.py -- scanning the module as text
# would pick up Python code (`len(`, `getattr(` ...).
from artemis.agents.operator import prompts as operator_prompts
for name in dir(operator_prompts):
value = getattr(operator_prompts, name)
if name.startswith("_") and isinstance(value, str) and len(value) > 80:
sources[f"prompts.py:{name}"] = value
return sources
def _referenced_tools() -> dict[str, set[str]]:
"""Returns {identifier: source labels} for every tool-shaped reference in scope."""
found: dict[str, set[str]] = {}
for label, text in _scoped_prompt_sources().items():
for match in _TOOL_REF.finditer(text):
name = match.group(1) or match.group(2)
found.setdefault(name, set()).add(label)
return found
# --- Prompt <-> manifest invariants --------------------------------------------------
def test_scan_finds_prompt_sources():
"""The scan must cover all expected sources; silent scope loss would void the suite."""
sources = _scoped_prompt_sources()
assert "flash_runner.md" in sources
assert "operator.json:main_template" in sources
assert "operator.json:troubleshooter_template" in sources
assert any(label.startswith("prompts.py:") for label in sources)
def test_every_prompt_tool_reference_is_classified():
"""Every tool referenced by an in-scope prompt must be known to the manifest."""
unclassified = {
name: sorted(labels)
for name, labels in _referenced_tools().items()
if name not in ALL_KNOWN_TOOLS and name not in NON_TOOL_IDENTIFIERS
}
assert not unclassified, (
"Prompts reference tools the manifest does not classify. Add each to the "
f"appropriate set in artemis/mcp/action_manifest.py: {unclassified}"
)
def test_required_actions_are_structurally_referenced():
"""A REQUIRED action must actually appear in a prompt; otherwise demote it."""
referenced = set(_referenced_tools())
unreferenced = REQUIRED_ACTIONS - referenced
assert not unreferenced, (
f"REQUIRED_ACTIONS not referenced by any prompt: {sorted(unreferenced)}. "
"Required status exists only to protect structural prompt dependencies."
)
def test_internal_actions_never_appear_in_prompts():
"""INTERNAL actions are adapter plumbing and must stay invisible to every model."""
leaked = INTERNAL_ACTIONS & set(_referenced_tools())
assert not leaked, f"Internal actions leaked into prompts: {sorted(leaked)}"
def test_manifest_sets_are_disjoint():
sets = {
"REQUIRED_ACTIONS": REQUIRED_ACTIONS,
"OPTIONAL_ACTIONS": OPTIONAL_ACTIONS,
"INTERNAL_ACTIONS": INTERNAL_ACTIONS,
"BACKEND_INDEPENDENT_TOOLS": BACKEND_INDEPENDENT_TOOLS,
}
names = list(sets)
for i, a in enumerate(names):
for b in names[i + 1 :]:
overlap = sets[a] & sets[b]
assert not overlap, f"{a} and {b} overlap: {sorted(overlap)}"
# --- Actuator contract ---------------------------------------------------------------
class _StubActuator:
"""Minimal actuator test double with configurable capabilities and extensions."""
def __init__(self, caps=None, exts=None):
self._caps = frozenset(caps if caps is not None else DEVICE_ACTIONS)
self._exts = list(exts or [])
def capabilities(self):
return self._caps
def extensions(self):
return self._exts
def _extension(**overrides) -> ExtensionTool:
async def _noop(**_kwargs):
return "ok"
defaults = dict(
name="calibrate_arm",
description=(
"Runs the robot arm's touch calibration sweep. Use before the first "
"physical action of a session, or after any missed tap."
),
input_schema={"type": "object", "properties": {}},
handler=_noop,
targets=frozenset({"operator", "flash"}),
)
defaults.update(overrides)
return ExtensionTool(**defaults)
def test_full_actuator_passes_validation():
validate_actuator(_StubActuator())
def test_minimal_actuator_passes_validation():
"""REQUIRED + INTERNAL alone is a legal backend; every optional action may be absent."""
validate_actuator(_StubActuator(caps=REQUIRED_ACTIONS | INTERNAL_ACTIONS))
def test_missing_required_action_fails_fast():
caps = (DEVICE_ACTIONS - {"click_sequence"})
with pytest.raises(ActuatorContractError, match="click_sequence"):
validate_actuator(_StubActuator(caps=caps))
def test_missing_internal_action_fails_fast():
caps = DEVICE_ACTIONS - {"observe_screen"}
with pytest.raises(ActuatorContractError, match="observe_screen"):
validate_actuator(_StubActuator(caps=caps))
def test_unknown_capability_is_rejected():
"""Backend-specific tools must go through extensions(), not capabilities()."""
with pytest.raises(ActuatorContractError, match="calibrate_arm"):
validate_actuator(_StubActuator(caps=DEVICE_ACTIONS | {"calibrate_arm"}))
def test_extension_name_collision_fails_fast():
with pytest.raises(ActuatorContractError, match="collides"):
validate_actuator(_StubActuator(exts=[_extension(name="click")]))
def test_duplicate_extension_names_fail_fast():
with pytest.raises(ActuatorContractError, match="more than once"):
validate_actuator(_StubActuator(exts=[_extension(), _extension()]))
def test_extension_with_thin_description_is_rejected():
"""Extensions get no prompt-level teaching; the description carries the contract."""
with pytest.raises(ActuatorContractError, match="description"):
validate_actuator(_StubActuator(exts=[_extension(description="Calibrates.")]))
def test_extension_with_unknown_target_is_rejected():
with pytest.raises(ActuatorContractError, match="unknown agent role"):
validate_actuator(_StubActuator(exts=[_extension(targets=frozenset({"planner"}))]))
# --- Surface derivation --------------------------------------------------------------
def _declarations(*names: str) -> list[ToolDeclaration]:
return [
ToolDeclaration(name=n, description=f"{n} tool", parameters={"type": "object"})
for n in names
]
def test_available_device_actions_excludes_internal():
actions = available_device_actions(_StubActuator())
assert actions == REQUIRED_ACTIONS | OPTIONAL_ACTIONS
assert not actions & INTERNAL_ACTIONS
def test_filter_drops_unimplemented_optional_actions():
actuator = _StubActuator(caps=DEVICE_ACTIONS - {"swipe", "manage_app"})
decls = _declarations("click", "swipe", "manage_app", "read_note")
kept = [d.name for d in filter_declarations(decls, actuator, agent="flash")]
assert kept == ["click", "read_note"]
def test_filter_passes_through_unclassified_agent_tools():
"""Agent-specific tools the manifest does not know must never be gated."""
actuator = _StubActuator(caps=REQUIRED_ACTIONS | INTERNAL_ACTIONS)
decls = _declarations("click_sequence", "reply_to_checker", "some_bespoke_tool")
kept = [d.name for d in filter_declarations(decls, actuator, agent="operator")]
assert kept == ["click_sequence", "reply_to_checker", "some_bespoke_tool"]
def test_filter_appends_extensions_for_targeted_agents_only():
actuator = _StubActuator(exts=[_extension(targets=frozenset({"flash"}))])
decls = _declarations("click")
flash = [d.name for d in filter_declarations(decls, actuator, agent="flash")]
operator = [d.name for d in filter_declarations(decls, actuator, agent="operator")]
assert flash == ["click", "calibrate_arm"]
assert operator == ["click"]
def test_filter_extension_declaration_shape():
"""Extension declarations must be indistinguishable in shape from known ones."""
actuator = _StubActuator(exts=[_extension()])
decls = filter_declarations([], actuator, agent="operator")
assert len(decls) == 1
ext_decl = decls[0]
assert isinstance(ext_decl, ToolDeclaration)
assert ext_decl.name == "calibrate_arm"
assert ext_decl.parameters == {"type": "object", "properties": {}}
def test_filter_rejects_unknown_agent_role():
with pytest.raises(ValueError, match="Unknown agent role"):
filter_declarations([], _StubActuator(), agent="planner")
def test_all_agent_roles_are_covered():
assert AGENT_ROLES == {"operator", "flash", "failure_analyzer", "validator"}
+156
View File
@@ -0,0 +1,156 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""End-to-end tests of the unified action server over the in-memory transport."""
import pytest
from artemis.mcp.action_manifest import (
ActuatorContractError,
DEVICE_ACTIONS,
INTERNAL_ACTIONS,
REQUIRED_ACTIONS,
ExtensionTool,
)
from artemis.mcp.action_server import build_action_server
from artemis.mcp.action_session import ActionSession
from artemis.mcp.actuators import MockActuator
pytestmark = pytest.mark.asyncio
async def _session_for(actuator) -> ActionSession:
return await ActionSession(build_action_server(actuator)).start()
async def test_click_end_to_end():
actuator = MockActuator(width=1000, height=2000)
session = await _session_for(actuator)
try:
res = await session.call("click", {"target": [500, 500]})
assert res.ok
assert res.message == "Clicked at [500, 500] (normalized) successfully."
assert actuator.action_history[-1]["action"] == "tap"
assert actuator.action_history[-1]["x"] == 500
assert actuator.action_history[-1]["y"] == 1000
finally:
await session.aclose()
async def test_device_refusal_is_not_a_protocol_error():
"""A failed action reaches the caller as ok=False, never as an exception."""
actuator = MockActuator()
session = await _session_for(actuator)
try:
res = await session.call("press_key", {"key": ""})
assert not res.ok
assert res.code.value == "INVALID_ARGS"
assert res.message == "Error executing key press ''."
finally:
await session.aclose()
async def test_partial_actuator_hides_optional_tools():
"""An unimplemented optional tool does not exist anywhere on the surface."""
minimal = MockActuator(capabilities=REQUIRED_ACTIONS | INTERNAL_ACTIONS)
session = await _session_for(minimal)
try:
listed = {t.name for t in (await session._session.list_tools()).tools}
assert "click_sequence" in listed
assert "observe_screen" in listed
for absent in ("click", "swipe", "manage_app", "wait_for_text", "open_link"):
assert absent not in listed
finally:
await session.aclose()
async def test_missing_required_action_fails_at_build_time():
broken = MockActuator(capabilities=DEVICE_ACTIONS - {"click_sequence"})
with pytest.raises(ActuatorContractError, match="click_sequence"):
build_action_server(broken)
async def test_extension_tool_is_served_and_callable():
async def calibrate(sweep_speed: int = 1) -> str:
return f"calibrated at speed {sweep_speed}"
ext = ExtensionTool(
name="calibrate_arm",
description=(
"Runs the robot arm's touch calibration sweep. Use before the first "
"physical action of a session, or after any missed tap."
),
input_schema={"type": "object", "properties": {}},
handler=calibrate,
targets=frozenset({"flash"}),
)
actuator = MockActuator(extensions=[ext])
session = await _session_for(actuator)
try:
listed = {t.name for t in (await session._session.list_tools()).tools}
assert "calibrate_arm" in listed
result = await session.call_raw("calibrate_arm", {"sweep_speed": 3})
text = next(b.text for b in result.content if b.type == "text")
assert text == "calibrated at speed 3"
finally:
await session.aclose()
async def test_extension_name_collision_fails_at_build_time():
async def bogus() -> str:
return "x"
ext = ExtensionTool(
name="click",
description="An extension improperly reusing a canonical tool name, which "
"must be rejected at build time.",
input_schema={"type": "object"},
handler=bogus,
)
with pytest.raises(ActuatorContractError, match="collides"):
build_action_server(MockActuator(extensions=[ext]))
async def test_unknown_tool_raises_protocol_error():
session = await _session_for(MockActuator())
try:
with pytest.raises(RuntimeError, match="not_a_tool"):
await session.call_raw("not_a_tool", {})
finally:
await session.aclose()
async def test_screenshot_and_hierarchy_helpers():
session = await _session_for(MockActuator(width=1000, height=2000))
try:
b64 = await session.screenshot_b64()
assert isinstance(b64, str) and len(b64) > 0
elements = await session.ui_hierarchy()
assert elements is not None
finally:
await session.aclose()
async def test_structured_content_carries_real_status():
"""The wire result exposes ok/code so no consumer ever sniffs substrings again."""
session = await _session_for(MockActuator())
try:
raw = await session.call_raw("click", {"target": [10, 10]})
sc = raw.structuredContent
assert sc["ok"] is True
assert sc["code"] == "OK"
assert sc["action"] == "click"
assert raw.isError is False
finally:
await session.aclose()
+188
View File
@@ -0,0 +1,188 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""ActionSession lifecycle tests, including the anyio same-task exit hazard."""
import asyncio
import pytest
from artemis.mcp.action_server import build_action_server
from artemis.mcp.action_session import ActionSession, get_action_session
from artemis.mcp.actuators import MockActuator
pytestmark = pytest.mark.asyncio
async def test_start_and_close():
session = ActionSession(build_action_server(MockActuator()))
await session.start()
assert session.started
await session.aclose()
assert not session.started
async def test_close_from_different_task():
"""The regression guard for the anyio cancel-scope same-task rule.
The in-memory transport's task group is entered inside the owner task; closing
from any *other* task must not raise ``RuntimeError: Attempted to exit cancel
scope in a different task``.
"""
session = ActionSession(build_action_server(MockActuator()))
await session.start()
async def _closer():
await session.aclose()
await asyncio.create_task(_closer())
assert not session.started
async def test_caller_timeout_does_not_brick_the_session():
"""Regression: cancelling a caller must never corrupt the shared transport.
Observed live on-device: ``asyncio.wait_for(session.ui_hierarchy(), 1.0)`` around
a slow hierarchy fetch cancelled ``call_tool`` mid-flight, corrupting the
in-memory streams -- every later call failed with an empty-message
``BrokenResourceError`` and the server never received another request. The owner
request queue makes an abandoned caller harmless; this test pins that down.
"""
actuator = MockActuator(width=1000, height=2000)
async def _slow_get_ui_elements():
await asyncio.sleep(2.0)
return []
actuator.get_ui_elements = _slow_get_ui_elements
session = await ActionSession(build_action_server(actuator)).start()
try:
with pytest.raises(asyncio.TimeoutError):
await asyncio.wait_for(session.ui_hierarchy(), timeout=0.1)
# The session must still be fully functional afterwards.
result = await session.call("click", {"target": [500, 500]})
assert result.ok
taps = [h for h in actuator.action_history if h["action"] == "tap"]
assert len(taps) == 1
finally:
await session.aclose()
async def test_per_call_timeout_fires_server_side_and_session_survives():
"""A per-call ``timeout`` runs inside the server tool, never on the transport.
mcp 1.29's ``read_timeout_seconds`` is NOT safe on the in-memory transport: a
late response to a timed-out request crashes the client receive loop
(``_handle_response`` sends into the request's already-closed response stream) --
observed live on-device as a session bricked after a single clean
``McpError`` timeout. Server-side timeouts produce an ordinary error payload
and no late response, so the session stays healthy by construction.
"""
actuator = MockActuator(width=1000, height=2000)
async def _slow_get_ui_elements():
await asyncio.sleep(2.0)
return []
actuator.get_ui_elements = _slow_get_ui_elements
session = await ActionSession(build_action_server(actuator)).start()
try:
with pytest.raises(RuntimeError, match="timed out"):
await session.ui_hierarchy(timeout=0.1)
result = await session.call("click", {"target": [500, 500]})
assert result.ok
finally:
await session.aclose()
async def test_transport_death_retires_session_for_rebuild():
"""A dead transport must flip ``started`` so get_action_session rebuilds.
Without this, one transport failure leaves the rest of a run failing on a
corpse (observed live: 24 minutes of empty-message ``ClosedResourceError``).
"""
import anyio
session = await ActionSession(build_action_server(MockActuator())).start()
real_call_tool = session._session.call_tool
async def _dying_call_tool(name, args, **kw):
raise anyio.ClosedResourceError()
session._session.call_tool = _dying_call_tool
with pytest.raises(anyio.ClosedResourceError):
await session.call("click", {"target": [1, 1]})
# Owner loop retires; the session reports itself unusable. The owner task may
# finish with an exception group from the transport teardown -- aclose() is the
# API that swallows that, so here we just wait for `started` to flip.
for _ in range(50):
if not session.started:
break
await asyncio.sleep(0.1)
assert not session.started
class Ctx:
action_session = session
fresh = await get_action_session(Ctx(), actuator=MockActuator())
assert fresh is not session
assert fresh.started
await fresh.aclose()
del real_call_tool
async def test_calls_are_serialized_by_the_lock():
"""Two concurrent calls must not interleave on the single device."""
actuator = MockActuator(width=1000, height=2000)
session = await ActionSession(build_action_server(actuator)).start()
try:
await asyncio.gather(
session.call("click", {"target": [100, 100]}),
session.call("click", {"target": [900, 900]}),
)
taps = [h for h in actuator.action_history if h["action"] == "tap"]
assert len(taps) == 2
finally:
await session.aclose()
async def test_call_before_start_raises():
session = ActionSession(build_action_server(MockActuator()))
with pytest.raises(RuntimeError, match="not started"):
await session.call("click", {"target": [1, 1]})
async def test_double_start_is_idempotent():
session = ActionSession(build_action_server(MockActuator()))
await session.start()
owner = session._owner
await session.start()
assert session._owner is owner
await session.aclose()
async def test_get_action_session_caches_on_ctx():
class Ctx:
action_session = None
ctx = Ctx()
actuator = MockActuator()
session = await get_action_session(ctx, actuator=actuator)
assert ctx.action_session is session
again = await get_action_session(ctx)
assert again is session
await session.aclose()
+146
View File
@@ -0,0 +1,146 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Actuator layer tests: coordinate conversion, message parity, driver dispatch."""
import pytest
from artemis.mcp.action_manifest import DEVICE_ACTIONS
from artemis.mcp.action_types import ActionCode
from artemis.mcp.actuators import Actuator, AdbActuator, MockActuator
@pytest.fixture
def actuator() -> MockActuator:
return MockActuator(width=1000, height=2000)
def test_mock_actuator_satisfies_protocol(actuator):
assert isinstance(actuator, Actuator)
assert isinstance(actuator, AdbActuator)
assert actuator.capabilities() == DEVICE_ACTIONS
assert actuator.extensions() == []
@pytest.mark.asyncio
async def test_click_converts_normalized_to_pixels(actuator):
res = await actuator.click(500, 500)
assert res.ok and res.code is ActionCode.OK
# Historical wording is part of the contract: transcripts and traces assert on it.
assert res.message == "Clicked at [500, 500] (normalized) successfully."
assert res.normalized_coordinates == [500, 500]
assert actuator.action_history[-1] == {
"action": "tap",
"x": 500,
"y": 1000,
"times": 1,
"duration_ms": 100,
}
@pytest.mark.asyncio
async def test_click_sequence_taps_each_point(actuator):
res = await actuator.click_sequence([(100, 100), (900, 900)], delay_ms=1)
assert res.ok
assert res.message == (
"Sequence clicked successfully: Tapped at [100, 200]; Tapped at [900, 1800]"
)
taps = [h for h in actuator.action_history if h["action"] == "tap"]
assert [(t["x"], t["y"]) for t in taps] == [(100, 200), (900, 1800)]
@pytest.mark.asyncio
async def test_long_press_message_and_duration(actuator):
res = await actuator.long_press(250, 750, duration_ms=1500)
assert res.ok
assert res.message == (
"Long pressed at [250, 750] (normalized) for 1500ms successfully."
)
assert res.duration_ms == 1500
@pytest.mark.asyncio
async def test_swipe_reports_normalized_endpoints(actuator):
res = await actuator.swipe((500, 800), (500, 200), 400)
assert res.ok
assert res.message == "Swipe completed successfully. Swiped from [500, 800] to [500, 200]."
assert res.normalized_coordinates == [500, 800, 500, 200]
@pytest.mark.asyncio
async def test_press_key_forwards_unknown_keycode_to_driver(actuator):
# Arbitrary keycodes (KEYCODE_* names, numeric codes) pass through to the driver,
# matching the historical adb_server behavior of accepting any Android key event.
res = await actuator.press_key("KEYCODE_DPAD_DOWN")
assert res.ok
assert res.message == "Executed key press 'KEYCODE_DPAD_DOWN'."
assert actuator.action_history[-1]["action"] == "press_key"
@pytest.mark.asyncio
async def test_press_key_rejects_empty_key(actuator):
res = await actuator.press_key("")
assert not res.ok
assert res.code is ActionCode.INVALID_ARGS
@pytest.mark.asyncio
async def test_input_text_append_moves_cursor_and_preserves_text(actuator, monkeypatch):
monkeypatch.setattr("asyncio.sleep", _instant_sleep)
res = await actuator.input_text("hello", (500, 300), clear_exist=False)
assert res.ok
assert res.message == "Executed typing 'hello'."
actions = [h["action"] for h in actuator.action_history]
# Append semantics: focus tap, cursor-to-end (KEYCODE_MOVE_END), then type without
# clearing — matching the historical adb_server focus_and_input_text contract.
assert actions == ["tap", "press_key", "input_text"]
typed = actuator.action_history[-1]
assert typed["clear_existing"] is False
cursor = actuator.action_history[-2]
assert cursor["key"] == "123"
@pytest.mark.asyncio
async def test_input_text_clear_erases_then_types_without_reclearing(actuator, monkeypatch):
monkeypatch.setattr("asyncio.sleep", _instant_sleep)
res = await actuator.input_text("hello", (500, 300), clear_exist=True)
assert res.ok
actions = [h["action"] for h in actuator.action_history]
# The controller's full clear runs as a shell keyevent batch on this driver.
assert actions == ["tap", "execute_shell", "input_text"]
assert actuator.action_history[-1]["clear_existing"] is False
async def _instant_sleep(_delay, *args, **kwargs):
return None
@pytest.mark.asyncio
async def test_wait_for_delay(actuator):
res = await actuator.wait_for_delay(1)
assert res.ok
assert res.message == "Waited for 1ms successfully."
def test_partial_capabilities_are_reported():
partial = MockActuator(capabilities=frozenset({"click_sequence"}))
assert partial.capabilities() == frozenset({"click_sequence"})
@pytest.mark.asyncio
async def test_observation_primitives(actuator):
shot = await actuator.take_screenshot()
assert isinstance(shot, str) and len(shot) > 0
data = await actuator.get_screen_data()
assert data.width == 1000 and data.height == 2000
+47 -18
View File
@@ -20,8 +20,10 @@ subprocess stdin isolation against pipe hijacking, and stdout purity.
import io
import json
import os
import queue
import subprocess
import sys
import threading
import time
from unittest.mock import MagicMock, patch
@@ -34,6 +36,31 @@ from artemis.utils.logger import get_logger
from mcp_server.utils import device_utils, env_utils
def _readline_with_timeout(pipe, timeout: float) -> str | None:
"""Read one line from a subprocess pipe with a timeout, portably.
select.select only supports sockets on Windows, so poll via a reader thread
instead of selecting on the pipe handle.
"""
result: queue.Queue = queue.Queue()
def _reader():
try:
result.put(pipe.readline())
except Exception:
result.put(b"")
reader = threading.Thread(target=_reader, daemon=True)
reader.start()
try:
line = result.get(timeout=timeout)
except queue.Empty:
return None
if not line:
return None
return line.decode("utf-8").strip()
def test_mcp_stdio_handshake_immediate_input():
"""Verify that MCP server over stdio responds to an immediate initialize request
@@ -51,6 +78,24 @@ def test_mcp_stdio_handshake_immediate_input():
"PYTHONUNBUFFERED": "1",
"PATH": os.environ.get("PATH", ""),
"PYTHONPATH": project_root,
# Windows: the interpreter's socket stack (Winsock/_overlapped),
# Path.home(), and tempfile handling need these system variables.
**{
key: os.environ[key]
for key in (
"SYSTEMROOT",
"SYSTEMDRIVE",
"WINDIR",
"TEMP",
"TMP",
"USERPROFILE",
"HOMEDRIVE",
"HOMEPATH",
"APPDATA",
"LOCALAPPDATA",
)
if key in os.environ
},
},
cwd=project_root,
)
@@ -71,15 +116,7 @@ def test_mcp_stdio_handshake_immediate_input():
p.stdin.flush()
# Read initialize response with a strict timeout
start = time.time()
init_resp_line = None
while time.time() - start < 6.0:
import select
r, _, _ = select.select([p.stdout], [], [], 0.2)
if r:
init_resp_line = p.stdout.readline().decode("utf-8").strip()
break
init_resp_line = _readline_with_timeout(p.stdout, 6.0)
assert init_resp_line is not None, "MCP server failed to respond to initialize request within 6 seconds (deadlock detected)!"
init_data = json.loads(init_resp_line)
@@ -93,15 +130,7 @@ def test_mcp_stdio_handshake_immediate_input():
p.stdin.write(json.dumps(tools_req).encode("utf-8") + b"\n")
p.stdin.flush()
tools_resp_line = None
start = time.time()
while time.time() - start < 4.0:
import select
r, _, _ = select.select([p.stdout], [], [], 0.2)
if r:
tools_resp_line = p.stdout.readline().decode("utf-8").strip()
break
tools_resp_line = _readline_with_timeout(p.stdout, 4.0)
assert tools_resp_line is not None, "MCP server failed to respond to tools/list request!"
tools_data = json.loads(tools_resp_line)
+97
View File
@@ -0,0 +1,97 @@
"""Tests for the spawn watchdog guarding mobile_run_task's standalone fallback."""
import json
import os
from unittest.mock import patch
from mcp_server.tools import task_runner as spawn_tool
def _write_status(trace_dir: str, trace_id: str, status: str) -> None:
os.makedirs(trace_dir, exist_ok=True)
with open(os.path.join(trace_dir, "status.json"), "w", encoding="utf-8") as f:
json.dump({"trace_id": trace_id, "status": status}, f)
def _patch_trace_dir(tmp_path):
return patch.object(spawn_tool.trace_store, "TRACES_DIR", str(tmp_path))
def test_watchdog_passes_when_runner_creates_logs(tmp_path):
trace_id = "wd-boot-ok"
trace_dir = tmp_path / trace_id
_write_status(str(trace_dir), trace_id, "running")
(trace_dir / "stdout.log").write_text("booted", encoding="utf-8")
with _patch_trace_dir(tmp_path), patch.object(spawn_tool, "_kill_process_tree") as kill:
assert spawn_tool._watch_spawn(
trace_id, pid=999999, queue_ticket="t", conversation_id=None,
deadline_seconds=0.2, poll_interval=0.05,
)
kill.assert_not_called()
def test_watchdog_skips_already_terminal_task(tmp_path):
trace_id = "wd-cancelled"
trace_dir = tmp_path / trace_id
_write_status(str(trace_dir), trace_id, "cancelled")
with _patch_trace_dir(tmp_path), patch.object(spawn_tool, "_kill_process_tree") as kill:
assert spawn_tool._watch_spawn(
trace_id, pid=999999, queue_ticket="t", conversation_id=None,
deadline_seconds=0.2, poll_interval=0.05,
)
kill.assert_not_called()
def test_watchdog_kills_and_fails_hung_runner(tmp_path):
trace_id = "wd-hung"
trace_dir = tmp_path / trace_id
_write_status(str(trace_dir), trace_id, "running")
with (
_patch_trace_dir(tmp_path),
patch.object(spawn_tool, "_kill_process_tree") as kill,
patch.object(spawn_tool.DeviceExecutionLock, "cancel_reservation") as cancel,
patch.object(spawn_tool, "notify") as notifier,
):
assert not spawn_tool._watch_spawn(
trace_id, pid=424242, queue_ticket="ticket-1", conversation_id="conv-1",
deadline_seconds=0.2, poll_interval=0.05,
)
kill.assert_called_once_with(424242)
cancel.assert_called_once_with("ticket-1")
notifier.assert_called_once()
with open(trace_dir / "status.json", encoding="utf-8") as f:
status = json.load(f)
assert status["status"] == "failed"
assert "no logs" in status.get("error", "")
def test_watchdog_tolerates_logs_appearing_at_deadline(tmp_path):
"""Logs that appear between the last poll and the deadline must not cause a kill."""
trace_id = "wd-late-boot"
trace_dir = tmp_path / trace_id
_write_status(str(trace_dir), trace_id, "running")
real_read_status = spawn_tool.trace_store.read_status
calls = {"n": 0}
def read_status_and_create_log(tid):
calls["n"] += 1
if calls["n"] == 1:
(trace_dir / "stderr.log").write_text("late boot", encoding="utf-8")
return real_read_status(tid)
with (
_patch_trace_dir(tmp_path),
patch.object(spawn_tool.trace_store, "read_status", side_effect=read_status_and_create_log),
patch.object(spawn_tool, "_kill_process_tree") as kill,
):
assert spawn_tool._watch_spawn(
trace_id, pid=999999, queue_ticket="t", conversation_id=None,
deadline_seconds=0.15, poll_interval=0.2,
)
kill.assert_not_called()
+133
View File
@@ -0,0 +1,133 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Regression tests for mobile_manage_task's state reconciliation.
Guards the invariants restored after the reconcile rewrite:
- a DB terminal verdict never overrides a status.json that already reached its
own terminal state (a completed task must keep its result),
- except to correct a previously liveness-inferred failure,
- and a task without a recorded pid (daemon dispatch) is never inferred dead.
"""
import sqlite3
import time
import pytest
import mcp_server.tools.task_manager as task_manager
from mcp_server.tools.task_manager import _LIVENESS_FAILURE_ERROR, _reconcile_task_state
@pytest.fixture
def no_persistence(monkeypatch):
"""Keep reconciliation in-memory: no DB, no status writes, no lock queries."""
monkeypatch.setattr(task_manager, "_find_data_engine_db", lambda: None)
monkeypatch.setattr(task_manager, "_session_tracked_by_lock", lambda _tid: False)
monkeypatch.setattr(task_manager.trace_store, "write_status", lambda *_a, **_k: None)
monkeypatch.setattr(
task_manager.trace_store, "update_trace_status", lambda *_a, **_k: None
)
monkeypatch.setattr(task_manager, "notify", lambda *_a, **_k: None)
def _db_with_session(tmp_path, trace_id: str, status: str, pid: int | None = 4242):
db_path = tmp_path / "artemis.db"
conn = sqlite3.connect(db_path)
conn.execute(
"CREATE TABLE sessions (session_id TEXT, status TEXT, pid INTEGER, start_time REAL)"
)
conn.execute(
"INSERT INTO sessions VALUES (?, ?, ?, ?)", (trace_id, status, pid, time.time())
)
conn.commit()
conn.close()
return str(db_path)
@pytest.fixture
def use_db(tmp_path, monkeypatch, no_persistence):
def _install(trace_id: str, status: str, pid: int | None = 4242):
db_path = _db_with_session(tmp_path, trace_id, status, pid)
monkeypatch.setattr(task_manager, "_find_data_engine_db", lambda: db_path)
return _install
def test_db_failed_never_overrides_completed_status(use_db):
use_db("t1", "failed")
status_data = {"status": "completed", "result": "the answer", "pid": 111}
current_status, _pid, is_alive = _reconcile_task_state("t1", status_data)
assert current_status == "completed"
assert status_data["status"] == "completed"
assert status_data["result"] == "the answer"
assert is_alive is False
def test_db_verdict_corrects_liveness_inferred_failure(use_db):
use_db("t2", "completed")
status_data = {"status": "failed", "error": _LIVENESS_FAILURE_ERROR, "pid": 111}
current_status, _pid, _is_alive = _reconcile_task_state("t2", status_data)
assert current_status == "completed"
assert status_data["error"] is None
def test_db_never_overrides_user_cancellation(use_db):
use_db("t3", "completed")
status_data = {"status": "cancelled", "pid": 111}
current_status, _pid, _is_alive = _reconcile_task_state("t3", status_data)
assert current_status == "cancelled"
def test_pidless_running_task_is_assumed_alive(no_persistence):
# Daemon dispatch writes no pid into status.json; liveness must not be
# inferred even long past the startup grace window.
status_data = {"status": "running", "start_time": time.time() - 3600}
current_status, _pid, is_alive = _reconcile_task_state("t4", status_data)
assert current_status == "running"
assert is_alive is True
assert status_data.get("error") is None
def test_dead_pid_past_grace_is_marked_failed(no_persistence, monkeypatch):
monkeypatch.setattr(task_manager, "_pid_alive", lambda _pid: False)
status_data = {
"status": "running",
"pid": 99999,
"start_time": time.time() - 3600,
}
current_status, _pid, is_alive = _reconcile_task_state("t5", status_data)
assert current_status == "failed"
assert status_data["error"] == _LIVENESS_FAILURE_ERROR
assert is_alive is False
def test_dead_pid_within_grace_is_assumed_alive(no_persistence, monkeypatch):
monkeypatch.setattr(task_manager, "_pid_alive", lambda _pid: False)
status_data = {"status": "running", "pid": 99999, "start_time": time.time()}
current_status, _pid, is_alive = _reconcile_task_state("t6", status_data)
assert current_status == "running"
assert is_alive is True
+4 -3
View File
@@ -14,6 +14,7 @@
"""Unit tests for MCP trace store."""
import os
import shutil
import tempfile
import uuid
@@ -80,6 +81,6 @@ def test_read_nonexistent_status(temp_trace_env):
def test_trace_paths(temp_trace_env):
trace_id = "test-paths-id"
trace_dir = trace_store.get_trace_dir(trace_id)
assert trace_store.get_trace_notes_dir(trace_id) == f"{trace_dir}/notes"
assert trace_store.get_trace_stdout_log_path(trace_id) == f"{trace_dir}/stdout.log"
assert trace_store.get_trace_stderr_log_path(trace_id) == f"{trace_dir}/stderr.log"
assert trace_store.get_trace_notes_dir(trace_id) == os.path.join(trace_dir, "notes")
assert trace_store.get_trace_stdout_log_path(trace_id) == os.path.join(trace_dir, "stdout.log")
assert trace_store.get_trace_stderr_log_path(trace_id) == os.path.join(trace_dir, "stderr.log")
+66 -1
View File
@@ -222,7 +222,9 @@ def test_cli_mcp_install_all(tmp_path, monkeypatch):
assert (tmp_path / ".gemini" / "rules" / "artemis.md").exists()
assert (tmp_path / ".cursorrules").exists()
assert (tmp_path / ".cursor" / "rules" / "artemis.mdc").exists()
assert (tmp_path / ".claude" / "CLAUDE.md").exists()
# Claude Code loads both CLAUDE.md and rules/*.md, so rules are installed
# only as the standalone rule file to avoid duplicated context.
assert not (tmp_path / ".claude" / "CLAUDE.md").exists()
assert (tmp_path / ".claude" / "rules" / "artemis.md").exists()
assert (tmp_path / ".codeium" / "windsurf" / "memories" / "global_rules.md").exists()
assert (tmp_path / ".codeium" / "windsurf" / "rules" / "artemis.md").exists()
@@ -236,6 +238,69 @@ def test_cli_mcp_install_all(tmp_path, monkeypatch):
assert (tmp_path / ".codex" / "AGENTS.md").exists()
def test_cli_mcp_install_claude_migrates_legacy_claude_md(tmp_path, monkeypatch):
"""Verify the claude target removes a previously injected CLAUDE.md block while preserving user content."""
monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path)
monkeypatch.setenv("APPDATA", str(tmp_path / "AppData" / "Roaming"))
monkeypatch.setattr("mcp_server.utils.env_utils.get_project_root", lambda: str(tmp_path))
claude_dir = tmp_path / ".claude"
claude_dir.mkdir(parents=True)
claude_md = claude_dir / "CLAUDE.md"
claude_md.write_text(
"# My own instructions\n\n"
"<!-- BEGIN ARTEMIS MOBILE TESTING RULES -->\nold injected rules\n"
"<!-- END ARTEMIS MOBILE TESTING RULES -->\n",
encoding="utf-8",
)
result = runner.invoke(app, ["mcp", "--install", "claude"])
assert result.exit_code == 0
remaining = claude_md.read_text(encoding="utf-8")
assert "My own instructions" in remaining
assert "ARTEMIS MOBILE TESTING RULES" not in remaining
rule_file = claude_dir / "rules" / "artemis.md"
assert rule_file.exists()
assert "Mobile Testing Mindset" in rule_file.read_text(encoding="utf-8")
def test_cli_mcp_install_claude_deletes_block_only_claude_md(tmp_path, monkeypatch):
"""Verify a CLAUDE.md consisting solely of the injected block is deleted outright."""
monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path)
monkeypatch.setenv("APPDATA", str(tmp_path / "AppData" / "Roaming"))
monkeypatch.setattr("mcp_server.utils.env_utils.get_project_root", lambda: str(tmp_path))
claude_dir = tmp_path / ".claude"
claude_dir.mkdir(parents=True)
claude_md = claude_dir / "CLAUDE.md"
claude_md.write_text(
"<!-- BEGIN ARTEMIS MOBILE TESTING RULES -->\nold injected rules\n"
"<!-- END ARTEMIS MOBILE TESTING RULES -->\n",
encoding="utf-8",
)
result = runner.invoke(app, ["mcp", "--install", "claude"])
assert result.exit_code == 0
assert not claude_md.exists()
assert (claude_dir / "rules" / "artemis.md").exists()
def test_cli_mcp_install_claude_refuses_unparseable_claude_json(tmp_path, monkeypatch):
"""Verify an unparseable ~/.claude.json is left untouched instead of being rewritten."""
monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path)
monkeypatch.setenv("APPDATA", str(tmp_path / "AppData" / "Roaming"))
monkeypatch.setattr("mcp_server.utils.env_utils.get_project_root", lambda: str(tmp_path))
corrupt = "{ this is not json at all"
claude_json = tmp_path / ".claude.json"
claude_json.write_text(corrupt, encoding="utf-8")
result = runner.invoke(app, ["mcp", "--install", "claude"])
assert result.exit_code == 0
assert claude_json.read_text(encoding="utf-8") == corrupt
def test_cli_mcp_install_codex_preserves_config_and_is_idempotent(tmp_path, monkeypatch):
"""Verify Codex TOML merging preserves other servers and updates only Artemis."""
monkeypatch.setattr("pathlib.Path.home", lambda: tmp_path)