feat: extract in-process MCP action layer, concurrent task dispatch, and spawn/reconcile hardening
- Extract device action execution into artemis/mcp action layer: typed ActionResult, capability manifest with startup validation, AdbActuator as the single physical implementation, in-memory ActionSession (replaces stdio subprocess MCP in validator), and McpActionExecutor preserving the legacy executor contract. adb_server keeps its external stdio/SSE surface unchanged and now reuses the actuator implementation; import-time side effects moved into configure_stdio_mode(). - Capability-gated prompt assembly (prompt_assembly.py): tool enums and teaching segments render per available backend tools; full-set output is byte-identical to the previous static text (locked by tests). - Admin console task queue: serial worker replaced by a per-device concurrent dispatcher (ARTEMIS_CONCURRENCY_MODE=global restores serial semantics), per-session cancellation, strict device binding on enqueue, unified stdio piping with tee to trace stdout.log, and shutdown cleanup of all active runs. Fix _pid_exists exception order so PermissionError is treated as alive. - mcp_server reliability: spawn watchdog kills runners stuck before interpreter startup and releases device reservations; task state reconcile uses psutil liveness, prefers runner-reported pids, and adds a 45s startup grace to stop false failure reports on Windows; conversation_id is now optional end-to-end; explicit device_serial is validated at submission. - CLI install now writes ~/.claude/rules/artemis.md for Claude Code and migrates/removes the legacy CLAUDE.md injection block; ~/.claude.json is never rewritten when it fails to parse. - Tests: Windows-portable stdio lifecycle and path assertions; new suites for action layer, prompt assembly, spawn watchdog, and task reconcile (895 passed, 3 skipped).
This commit is contained in:
weilinliu55 committed
1 parent
60312abbb0
commit
cbdca1497c
52 files changed
+5735
-1649
No files matched your search
@@ -0,0 +1,419 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""ADB-backed actuator: the reference implementation of the actuator contract.
|
||||
|
||||
The device-call bodies here were extracted from
|
||||
``artemis.agents.validator.tool_declarations.MobileActionExecutor``; the outcome
|
||||
message wording is preserved byte-for-byte because agent transcripts, trace payloads,
|
||||
and existing tests assert on it.
|
||||
|
||||
Error convention: device-level refusals the controller *reports* (an ``error`` field,
|
||||
a falsy success flag) come back as ``ActionResult(ok=False)`` with the historical
|
||||
message. Unexpected exceptions are allowed to propagate -- the caller (executor or MCP
|
||||
server layer) owns the ``"Error during X: {e}"`` wrapping, exactly as the original
|
||||
executor's ``try/except`` did.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import inspect
|
||||
import re
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
from artemis.context import ArtemisContext
|
||||
from artemis.controllers.unified_controller import UnifiedMobileController
|
||||
from artemis.mcp.action_manifest import DEVICE_ACTIONS, ExtensionTool
|
||||
from artemis.mcp.action_types import ActionCode, ActionResult
|
||||
from artemis.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
__all__ = ["AdbActuator", "find_element_at_coords", "ensure_focus_at_coords"]
|
||||
|
||||
|
||||
def find_element_at_coords(elements: list[dict], x: int, y: int) -> dict | None:
|
||||
"""Finds the smallest (leaf-most) focusable element containing pixel [x, y].
|
||||
|
||||
Moved from ``artemis.mcp.adb_server`` so both the stdio server and in-process
|
||||
actuator share one implementation.
|
||||
"""
|
||||
matching_element = None
|
||||
min_area = float("inf")
|
||||
|
||||
for elem in elements:
|
||||
is_focusable = (
|
||||
elem.get("focusable") == "true"
|
||||
or elem.get("clickable") == "true"
|
||||
or "EditText" in str(elem.get("class", ""))
|
||||
)
|
||||
if not is_focusable:
|
||||
continue
|
||||
|
||||
bounds_str = elem.get("bounds")
|
||||
if bounds_str and isinstance(bounds_str, str):
|
||||
match = re.match(r"\[(\-?\d+),(\-?\d+)\]\[(\-?\d+),(\-?\d+)\]", bounds_str)
|
||||
if match:
|
||||
x1, y1, x2, y2 = map(int, match.groups())
|
||||
if x1 <= x <= x2 and y1 <= y <= y2:
|
||||
area = (x2 - x1) * (y2 - y1)
|
||||
if area < min_area:
|
||||
min_area = area
|
||||
matching_element = elem
|
||||
|
||||
return matching_element
|
||||
|
||||
|
||||
async def ensure_focus_at_coords(controller, x: int, y: int) -> str | None:
|
||||
"""Ensures the element at pixel [x, y] is focused, tapping only if needed.
|
||||
|
||||
Returns an error string on failure, ``None`` on success. Moved from
|
||||
``artemis.mcp.adb_server``.
|
||||
"""
|
||||
try:
|
||||
elements = await controller.get_ui_elements()
|
||||
elem = find_element_at_coords(elements, x, y)
|
||||
if elem and elem.get("focused") == "true":
|
||||
logger.info(f"Element under [{x}, {y}] is already focused. Skipping tap.")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to check focus status: {e}. Falling back to unconditional tap.")
|
||||
|
||||
result = await controller.tap_at(x=x, y=y)
|
||||
if hasattr(result, "error") and result.error:
|
||||
return result.error
|
||||
|
||||
# Wait for the UI to settle and keyboard to pop up after tapping
|
||||
await asyncio.sleep(1.0)
|
||||
return None
|
||||
|
||||
|
||||
class AdbActuator:
|
||||
"""Drives an Android device through ``UnifiedMobileController`` over ADB."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
ctx: ArtemisContext,
|
||||
controller: UnifiedMobileController | None = None,
|
||||
):
|
||||
self.ctx = ctx
|
||||
self.controller = controller or UnifiedMobileController(ctx)
|
||||
|
||||
# --- Contract --------------------------------------------------------------------
|
||||
|
||||
def capabilities(self) -> frozenset[str]:
|
||||
return DEVICE_ACTIONS
|
||||
|
||||
def extensions(self) -> list[ExtensionTool]:
|
||||
return []
|
||||
|
||||
# --- Coordinate helpers ----------------------------------------------------------
|
||||
|
||||
def _dims(self) -> tuple[int, int]:
|
||||
width = getattr(self.ctx.device, "device_width", 1080) if self.ctx.device else 1080
|
||||
height = getattr(self.ctx.device, "device_height", 2400) if self.ctx.device else 2400
|
||||
return width or 1080, height or 2400
|
||||
|
||||
def _to_px(self, nx: int, ny: int) -> tuple[int, int]:
|
||||
width, height = self._dims()
|
||||
x = int(max(0, min(width - 1, int(nx) * width / 1000)))
|
||||
y = int(max(0, min(height - 1, int(ny) * height / 1000)))
|
||||
return x, y
|
||||
|
||||
# --- Required action -------------------------------------------------------------
|
||||
|
||||
async def click_sequence(
|
||||
self, points: list[tuple[int, int]], delay_ms: int = 50
|
||||
) -> ActionResult:
|
||||
outcomes = []
|
||||
for i, (nx, ny) in enumerate(points):
|
||||
x, y = self._to_px(nx, ny)
|
||||
result = await self.controller.tap_at(x, y)
|
||||
err = getattr(result, "error", None) if result else None
|
||||
if err:
|
||||
return ActionResult.failure(
|
||||
"click_sequence",
|
||||
f"Error executing click at step {i + 1}: {err}",
|
||||
detail=str(err),
|
||||
)
|
||||
outcomes.append(f"Tapped at [{x}, {y}]")
|
||||
if i < len(points) - 1:
|
||||
await asyncio.sleep(max(0, delay_ms) / 1000.0)
|
||||
|
||||
return ActionResult.success(
|
||||
"click_sequence",
|
||||
f"Sequence clicked successfully: {'; '.join(outcomes)}",
|
||||
)
|
||||
|
||||
# --- Optional actions ------------------------------------------------------------
|
||||
|
||||
async def click(
|
||||
self, nx: int, ny: int, times: int = 1, delay_ms: int = 100
|
||||
) -> ActionResult:
|
||||
x, y = self._to_px(nx, ny)
|
||||
result = await self.controller.tap_at(x, y, times=times, delay_ms=delay_ms)
|
||||
err = getattr(result, "error", None) if result else None
|
||||
if err is not None:
|
||||
return ActionResult.failure(
|
||||
"click", f"Error executing click: {err}", detail=str(err)
|
||||
)
|
||||
return ActionResult.success(
|
||||
"click",
|
||||
f"Clicked at [{nx}, {ny}] (normalized) successfully.",
|
||||
normalized_coordinates=[int(nx), int(ny)],
|
||||
)
|
||||
|
||||
async def long_press(self, nx: int, ny: int, duration_ms: int = 1000) -> ActionResult:
|
||||
x, y = self._to_px(nx, ny)
|
||||
result = await self.controller.tap_at(
|
||||
x, y, long_press=True, long_press_duration=duration_ms
|
||||
)
|
||||
err = getattr(result, "error", None) if result else None
|
||||
if err is not None:
|
||||
return ActionResult.failure(
|
||||
"long_press", f"Error executing long press: {err}", detail=str(err)
|
||||
)
|
||||
return ActionResult.success(
|
||||
"long_press",
|
||||
f"Long pressed at [{nx}, {ny}] (normalized) for {duration_ms}ms successfully.",
|
||||
normalized_coordinates=[int(nx), int(ny)],
|
||||
duration_ms=duration_ms,
|
||||
)
|
||||
|
||||
async def input_text(
|
||||
self,
|
||||
text: str,
|
||||
target: tuple[int, int] | None = None,
|
||||
clear_exist: bool = True,
|
||||
) -> ActionResult:
|
||||
coords = None
|
||||
if target:
|
||||
nx, ny = target
|
||||
x, y = self._to_px(nx, ny)
|
||||
coords = [int(nx), int(ny)]
|
||||
err = await ensure_focus_at_coords(self.controller, x, y)
|
||||
if err:
|
||||
return ActionResult.failure(
|
||||
"input_text",
|
||||
f"Error focusing element: {err}",
|
||||
detail=str(err),
|
||||
normalized_coordinates=coords,
|
||||
)
|
||||
if clear_exist:
|
||||
if hasattr(self.controller, "erase_text") and not await self.controller.erase_text():
|
||||
return ActionResult.failure(
|
||||
"input_text",
|
||||
"Failed to clear existing text",
|
||||
normalized_coordinates=coords,
|
||||
)
|
||||
elif hasattr(self.controller, "press_key"):
|
||||
# KEYCODE_MOVE_END: move cursor to the end for reliable append.
|
||||
await self.controller.press_key("123")
|
||||
if hasattr(self.controller, "type_text") and not await self.controller.type_text(
|
||||
text, clear_existing=False
|
||||
):
|
||||
return ActionResult.failure(
|
||||
"input_text",
|
||||
f"Failed typing '{text}'.",
|
||||
normalized_coordinates=coords,
|
||||
)
|
||||
|
||||
return ActionResult.success(
|
||||
"input_text",
|
||||
f"Executed typing '{text}'.",
|
||||
normalized_coordinates=coords,
|
||||
)
|
||||
|
||||
async def swipe(
|
||||
self,
|
||||
start: tuple[int, int],
|
||||
end: tuple[int, int],
|
||||
duration_ms: int = 800,
|
||||
) -> ActionResult:
|
||||
nx1, ny1 = start
|
||||
nx2, ny2 = end
|
||||
px1, py1 = self._to_px(nx1, ny1)
|
||||
px2, py2 = self._to_px(nx2, ny2)
|
||||
err = await self.controller.swipe_coords(px1, py1, px2, py2, duration_ms)
|
||||
if err:
|
||||
return ActionResult.failure("swipe", f"Error dragging: {err}", detail=str(err))
|
||||
return ActionResult.success(
|
||||
"swipe",
|
||||
f"Swipe completed successfully. Swiped from [{nx1}, {ny1}] to [{nx2}, {ny2}].",
|
||||
normalized_coordinates=[int(nx1), int(ny1), int(nx2), int(ny2)],
|
||||
duration_ms=duration_ms,
|
||||
)
|
||||
|
||||
async def press_key(self, key: str) -> ActionResult:
|
||||
key_str = str(key).lower()
|
||||
if not key_str:
|
||||
return ActionResult.failure(
|
||||
"press_key",
|
||||
f"Error executing key press '{key}'.",
|
||||
code=ActionCode.INVALID_ARGS,
|
||||
)
|
||||
|
||||
if key_str == "back" and hasattr(self.controller, "go_back"):
|
||||
await self.controller.go_back()
|
||||
elif key_str == "home" and hasattr(self.controller, "go_home"):
|
||||
await self.controller.go_home()
|
||||
elif key_str == "enter" and hasattr(self.controller, "press_enter"):
|
||||
await self.controller.press_enter()
|
||||
elif hasattr(self.controller, "press_key"):
|
||||
# Pass unrecognized keycodes through verbatim (KEYCODE_* names, numeric
|
||||
# codes): the driver resolves or forwards them, matching the historical
|
||||
# adb_server behavior of accepting any Android key event.
|
||||
res = await self.controller.press_key(key)
|
||||
err = getattr(res, "error", None) if res else None
|
||||
if err:
|
||||
return ActionResult.failure(
|
||||
"press_key",
|
||||
f"Error executing key press '{key}': {err}",
|
||||
detail=str(err),
|
||||
)
|
||||
if not res:
|
||||
return ActionResult.failure(
|
||||
"press_key", f"Error executing key press '{key}'."
|
||||
)
|
||||
|
||||
return ActionResult.success("press_key", f"Executed key press '{key}'.")
|
||||
|
||||
async def manage_app(self, action: str, app_name: str) -> ActionResult:
|
||||
# Imported lazily: launch_app pulls in tool wrappers that are costly at import.
|
||||
from artemis.tools.mobile.launch_app import find_package
|
||||
from artemis.utils.app_launch_utils import launch_app_with_retries
|
||||
|
||||
res = find_package(self.ctx, app_name, use_fallback=False)
|
||||
pkg = await res if inspect.iscoroutine(res) else res
|
||||
if not pkg:
|
||||
return ActionResult.failure(
|
||||
"manage_app",
|
||||
f"Error finding package for app: {app_name}",
|
||||
code=ActionCode.PACKAGE_NOT_FOUND,
|
||||
)
|
||||
target_pkg = pkg
|
||||
|
||||
if action.lower() == "launch":
|
||||
res_launch = launch_app_with_retries(self.ctx, target_pkg)
|
||||
if inspect.iscoroutine(res_launch):
|
||||
res_launch = await res_launch
|
||||
if isinstance(res_launch, tuple):
|
||||
success, error_msg = res_launch
|
||||
else:
|
||||
success, error_msg = bool(res_launch), ""
|
||||
|
||||
if success:
|
||||
return ActionResult.success(
|
||||
"manage_app",
|
||||
f"Launched app '{app_name}' ({target_pkg}) successfully.",
|
||||
)
|
||||
return ActionResult.failure(
|
||||
"manage_app",
|
||||
f"Failed to launch app '{app_name}': {error_msg}",
|
||||
detail=error_msg or None,
|
||||
)
|
||||
elif action.lower() == "stop":
|
||||
if hasattr(self.controller, "terminate_app"):
|
||||
res_term = self.controller.terminate_app(target_pkg)
|
||||
if inspect.iscoroutine(res_term):
|
||||
await res_term
|
||||
return ActionResult.success(
|
||||
"manage_app", f"Terminated app '{app_name}' successfully."
|
||||
)
|
||||
return ActionResult.failure(
|
||||
"manage_app",
|
||||
f"Invalid manage_app action: {action}",
|
||||
code=ActionCode.INVALID_ARGS,
|
||||
)
|
||||
|
||||
async def wait_for_delay(self, time_in_ms: int) -> ActionResult:
|
||||
delay_s = max(0.0, float(time_in_ms) / 1000.0)
|
||||
await asyncio.sleep(delay_s)
|
||||
return ActionResult.success(
|
||||
"wait_for_delay",
|
||||
f"Waited for {time_in_ms}ms successfully.",
|
||||
duration_ms=int(time_in_ms),
|
||||
)
|
||||
|
||||
async def wait_for_text(
|
||||
self, text: str, wait_state: str | None = None, timeout_ms: int | None = None
|
||||
) -> ActionResult:
|
||||
timeout_s = (timeout_ms or 5000) / 1000.0
|
||||
start = time.time()
|
||||
found = False
|
||||
target_state = (wait_state or "appear").lower()
|
||||
|
||||
while time.time() - start < timeout_s:
|
||||
tree = await self.controller.get_ui_elements()
|
||||
text_present = text.lower() in str(tree or "").lower()
|
||||
if target_state == "appear" and text_present:
|
||||
found = True
|
||||
break
|
||||
elif target_state == "disappear" and not text_present:
|
||||
found = True
|
||||
break
|
||||
await asyncio.sleep(0.5)
|
||||
|
||||
if found:
|
||||
return ActionResult.success(
|
||||
"wait_for_text",
|
||||
f"Successfully waited for text '{text}' to {target_state}.",
|
||||
)
|
||||
return ActionResult.failure(
|
||||
"wait_for_text",
|
||||
f"Timed out waiting for text '{text}' to {target_state}.",
|
||||
code=ActionCode.TIMEOUT,
|
||||
)
|
||||
|
||||
async def open_link(self, url: str) -> ActionResult:
|
||||
success = await self.controller.open_url(url)
|
||||
if success:
|
||||
return ActionResult.success("open_link", f"Opened link '{url}' successfully.")
|
||||
return ActionResult.failure("open_link", f"Failed to open link '{url}'.")
|
||||
|
||||
async def erase_one_char(self) -> ActionResult:
|
||||
success = await self.controller.erase_text(nb_chars=1)
|
||||
if success:
|
||||
return ActionResult.success("erase_one_char", "Erased one character.")
|
||||
return ActionResult.failure("erase_one_char", "Failed to erase one character.")
|
||||
|
||||
async def focus_and_clear_text(self, nx: int, ny: int) -> ActionResult:
|
||||
x, y = self._to_px(nx, ny)
|
||||
err = await ensure_focus_at_coords(self.controller, x, y)
|
||||
if err:
|
||||
return ActionResult.failure(
|
||||
"focus_and_clear_text",
|
||||
f"Error focusing element: {err}",
|
||||
detail=str(err),
|
||||
normalized_coordinates=[int(nx), int(ny)],
|
||||
)
|
||||
success = await self.controller.erase_text()
|
||||
if success:
|
||||
return ActionResult.success(
|
||||
"focus_and_clear_text",
|
||||
f"Cleared text at [{nx}, {ny}] (normalized) successfully.",
|
||||
normalized_coordinates=[int(nx), int(ny)],
|
||||
)
|
||||
return ActionResult.failure("focus_and_clear_text", "Failed to erase text.")
|
||||
|
||||
# --- Internal observation primitives ---------------------------------------------
|
||||
|
||||
async def take_screenshot(self) -> str:
|
||||
return await self.controller.take_screenshot()
|
||||
|
||||
async def get_ui_elements(self) -> Any:
|
||||
return await self.controller.get_ui_elements()
|
||||
|
||||
async def get_screen_data(self) -> Any:
|
||||
return await self.controller.get_screen_data()
|
||||
Reference in new issue
Block a user