weilinliu55
cbdca1497c
- Extract device action execution into artemis/mcp action layer: typed ActionResult, capability manifest with startup validation, AdbActuator as the single physical implementation, in-memory ActionSession (replaces stdio subprocess MCP in validator), and McpActionExecutor preserving the legacy executor contract. adb_server keeps its external stdio/SSE surface unchanged and now reuses the actuator implementation; import-time side effects moved into configure_stdio_mode(). - Capability-gated prompt assembly (prompt_assembly.py): tool enums and teaching segments render per available backend tools; full-set output is byte-identical to the previous static text (locked by tests). - Admin console task queue: serial worker replaced by a per-device concurrent dispatcher (ARTEMIS_CONCURRENCY_MODE=global restores serial semantics), per-session cancellation, strict device binding on enqueue, unified stdio piping with tee to trace stdout.log, and shutdown cleanup of all active runs. Fix _pid_exists exception order so PermissionError is treated as alive. - mcp_server reliability: spawn watchdog kills runners stuck before interpreter startup and releases device reservations; task state reconcile uses psutil liveness, prefers runner-reported pids, and adds a 45s startup grace to stop false failure reports on Windows; conversation_id is now optional end-to-end; explicit device_serial is validated at submission. - CLI install now writes ~/.claude/rules/artemis.md for Claude Code and migrates/removes the legacy CLAUDE.md injection block; ~/.claude.json is never rewritten when it fails to parse. - Tests: Windows-portable stdio lifecycle and path assertions; new suites for action layer, prompt assembly, spawn watchdog, and task reconcile (895 passed, 3 skipped).
420 lines
16 KiB
Python
420 lines
16 KiB
Python
# Copyright 2026 Google LLC
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""ADB-backed actuator: the reference implementation of the actuator contract.
|
|
|
|
The device-call bodies here were extracted from
|
|
``artemis.agents.validator.tool_declarations.MobileActionExecutor``; the outcome
|
|
message wording is preserved byte-for-byte because agent transcripts, trace payloads,
|
|
and existing tests assert on it.
|
|
|
|
Error convention: device-level refusals the controller *reports* (an ``error`` field,
|
|
a falsy success flag) come back as ``ActionResult(ok=False)`` with the historical
|
|
message. Unexpected exceptions are allowed to propagate -- the caller (executor or MCP
|
|
server layer) owns the ``"Error during X: {e}"`` wrapping, exactly as the original
|
|
executor's ``try/except`` did.
|
|
"""
|
|
|
|
import asyncio
|
|
import inspect
|
|
import re
|
|
import time
|
|
from typing import Any
|
|
|
|
from artemis.context import ArtemisContext
|
|
from artemis.controllers.unified_controller import UnifiedMobileController
|
|
from artemis.mcp.action_manifest import DEVICE_ACTIONS, ExtensionTool
|
|
from artemis.mcp.action_types import ActionCode, ActionResult
|
|
from artemis.utils.logger import get_logger
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
__all__ = ["AdbActuator", "find_element_at_coords", "ensure_focus_at_coords"]
|
|
|
|
|
|
def find_element_at_coords(elements: list[dict], x: int, y: int) -> dict | None:
|
|
"""Finds the smallest (leaf-most) focusable element containing pixel [x, y].
|
|
|
|
Moved from ``artemis.mcp.adb_server`` so both the stdio server and in-process
|
|
actuator share one implementation.
|
|
"""
|
|
matching_element = None
|
|
min_area = float("inf")
|
|
|
|
for elem in elements:
|
|
is_focusable = (
|
|
elem.get("focusable") == "true"
|
|
or elem.get("clickable") == "true"
|
|
or "EditText" in str(elem.get("class", ""))
|
|
)
|
|
if not is_focusable:
|
|
continue
|
|
|
|
bounds_str = elem.get("bounds")
|
|
if bounds_str and isinstance(bounds_str, str):
|
|
match = re.match(r"\[(\-?\d+),(\-?\d+)\]\[(\-?\d+),(\-?\d+)\]", bounds_str)
|
|
if match:
|
|
x1, y1, x2, y2 = map(int, match.groups())
|
|
if x1 <= x <= x2 and y1 <= y <= y2:
|
|
area = (x2 - x1) * (y2 - y1)
|
|
if area < min_area:
|
|
min_area = area
|
|
matching_element = elem
|
|
|
|
return matching_element
|
|
|
|
|
|
async def ensure_focus_at_coords(controller, x: int, y: int) -> str | None:
|
|
"""Ensures the element at pixel [x, y] is focused, tapping only if needed.
|
|
|
|
Returns an error string on failure, ``None`` on success. Moved from
|
|
``artemis.mcp.adb_server``.
|
|
"""
|
|
try:
|
|
elements = await controller.get_ui_elements()
|
|
elem = find_element_at_coords(elements, x, y)
|
|
if elem and elem.get("focused") == "true":
|
|
logger.info(f"Element under [{x}, {y}] is already focused. Skipping tap.")
|
|
return None
|
|
except Exception as e:
|
|
logger.warning(f"Failed to check focus status: {e}. Falling back to unconditional tap.")
|
|
|
|
result = await controller.tap_at(x=x, y=y)
|
|
if hasattr(result, "error") and result.error:
|
|
return result.error
|
|
|
|
# Wait for the UI to settle and keyboard to pop up after tapping
|
|
await asyncio.sleep(1.0)
|
|
return None
|
|
|
|
|
|
class AdbActuator:
|
|
"""Drives an Android device through ``UnifiedMobileController`` over ADB."""
|
|
|
|
def __init__(
|
|
self,
|
|
ctx: ArtemisContext,
|
|
controller: UnifiedMobileController | None = None,
|
|
):
|
|
self.ctx = ctx
|
|
self.controller = controller or UnifiedMobileController(ctx)
|
|
|
|
# --- Contract --------------------------------------------------------------------
|
|
|
|
def capabilities(self) -> frozenset[str]:
|
|
return DEVICE_ACTIONS
|
|
|
|
def extensions(self) -> list[ExtensionTool]:
|
|
return []
|
|
|
|
# --- Coordinate helpers ----------------------------------------------------------
|
|
|
|
def _dims(self) -> tuple[int, int]:
|
|
width = getattr(self.ctx.device, "device_width", 1080) if self.ctx.device else 1080
|
|
height = getattr(self.ctx.device, "device_height", 2400) if self.ctx.device else 2400
|
|
return width or 1080, height or 2400
|
|
|
|
def _to_px(self, nx: int, ny: int) -> tuple[int, int]:
|
|
width, height = self._dims()
|
|
x = int(max(0, min(width - 1, int(nx) * width / 1000)))
|
|
y = int(max(0, min(height - 1, int(ny) * height / 1000)))
|
|
return x, y
|
|
|
|
# --- Required action -------------------------------------------------------------
|
|
|
|
async def click_sequence(
|
|
self, points: list[tuple[int, int]], delay_ms: int = 50
|
|
) -> ActionResult:
|
|
outcomes = []
|
|
for i, (nx, ny) in enumerate(points):
|
|
x, y = self._to_px(nx, ny)
|
|
result = await self.controller.tap_at(x, y)
|
|
err = getattr(result, "error", None) if result else None
|
|
if err:
|
|
return ActionResult.failure(
|
|
"click_sequence",
|
|
f"Error executing click at step {i + 1}: {err}",
|
|
detail=str(err),
|
|
)
|
|
outcomes.append(f"Tapped at [{x}, {y}]")
|
|
if i < len(points) - 1:
|
|
await asyncio.sleep(max(0, delay_ms) / 1000.0)
|
|
|
|
return ActionResult.success(
|
|
"click_sequence",
|
|
f"Sequence clicked successfully: {'; '.join(outcomes)}",
|
|
)
|
|
|
|
# --- Optional actions ------------------------------------------------------------
|
|
|
|
async def click(
|
|
self, nx: int, ny: int, times: int = 1, delay_ms: int = 100
|
|
) -> ActionResult:
|
|
x, y = self._to_px(nx, ny)
|
|
result = await self.controller.tap_at(x, y, times=times, delay_ms=delay_ms)
|
|
err = getattr(result, "error", None) if result else None
|
|
if err is not None:
|
|
return ActionResult.failure(
|
|
"click", f"Error executing click: {err}", detail=str(err)
|
|
)
|
|
return ActionResult.success(
|
|
"click",
|
|
f"Clicked at [{nx}, {ny}] (normalized) successfully.",
|
|
normalized_coordinates=[int(nx), int(ny)],
|
|
)
|
|
|
|
async def long_press(self, nx: int, ny: int, duration_ms: int = 1000) -> ActionResult:
|
|
x, y = self._to_px(nx, ny)
|
|
result = await self.controller.tap_at(
|
|
x, y, long_press=True, long_press_duration=duration_ms
|
|
)
|
|
err = getattr(result, "error", None) if result else None
|
|
if err is not None:
|
|
return ActionResult.failure(
|
|
"long_press", f"Error executing long press: {err}", detail=str(err)
|
|
)
|
|
return ActionResult.success(
|
|
"long_press",
|
|
f"Long pressed at [{nx}, {ny}] (normalized) for {duration_ms}ms successfully.",
|
|
normalized_coordinates=[int(nx), int(ny)],
|
|
duration_ms=duration_ms,
|
|
)
|
|
|
|
async def input_text(
|
|
self,
|
|
text: str,
|
|
target: tuple[int, int] | None = None,
|
|
clear_exist: bool = True,
|
|
) -> ActionResult:
|
|
coords = None
|
|
if target:
|
|
nx, ny = target
|
|
x, y = self._to_px(nx, ny)
|
|
coords = [int(nx), int(ny)]
|
|
err = await ensure_focus_at_coords(self.controller, x, y)
|
|
if err:
|
|
return ActionResult.failure(
|
|
"input_text",
|
|
f"Error focusing element: {err}",
|
|
detail=str(err),
|
|
normalized_coordinates=coords,
|
|
)
|
|
if clear_exist:
|
|
if hasattr(self.controller, "erase_text") and not await self.controller.erase_text():
|
|
return ActionResult.failure(
|
|
"input_text",
|
|
"Failed to clear existing text",
|
|
normalized_coordinates=coords,
|
|
)
|
|
elif hasattr(self.controller, "press_key"):
|
|
# KEYCODE_MOVE_END: move cursor to the end for reliable append.
|
|
await self.controller.press_key("123")
|
|
if hasattr(self.controller, "type_text") and not await self.controller.type_text(
|
|
text, clear_existing=False
|
|
):
|
|
return ActionResult.failure(
|
|
"input_text",
|
|
f"Failed typing '{text}'.",
|
|
normalized_coordinates=coords,
|
|
)
|
|
|
|
return ActionResult.success(
|
|
"input_text",
|
|
f"Executed typing '{text}'.",
|
|
normalized_coordinates=coords,
|
|
)
|
|
|
|
async def swipe(
|
|
self,
|
|
start: tuple[int, int],
|
|
end: tuple[int, int],
|
|
duration_ms: int = 800,
|
|
) -> ActionResult:
|
|
nx1, ny1 = start
|
|
nx2, ny2 = end
|
|
px1, py1 = self._to_px(nx1, ny1)
|
|
px2, py2 = self._to_px(nx2, ny2)
|
|
err = await self.controller.swipe_coords(px1, py1, px2, py2, duration_ms)
|
|
if err:
|
|
return ActionResult.failure("swipe", f"Error dragging: {err}", detail=str(err))
|
|
return ActionResult.success(
|
|
"swipe",
|
|
f"Swipe completed successfully. Swiped from [{nx1}, {ny1}] to [{nx2}, {ny2}].",
|
|
normalized_coordinates=[int(nx1), int(ny1), int(nx2), int(ny2)],
|
|
duration_ms=duration_ms,
|
|
)
|
|
|
|
async def press_key(self, key: str) -> ActionResult:
|
|
key_str = str(key).lower()
|
|
if not key_str:
|
|
return ActionResult.failure(
|
|
"press_key",
|
|
f"Error executing key press '{key}'.",
|
|
code=ActionCode.INVALID_ARGS,
|
|
)
|
|
|
|
if key_str == "back" and hasattr(self.controller, "go_back"):
|
|
await self.controller.go_back()
|
|
elif key_str == "home" and hasattr(self.controller, "go_home"):
|
|
await self.controller.go_home()
|
|
elif key_str == "enter" and hasattr(self.controller, "press_enter"):
|
|
await self.controller.press_enter()
|
|
elif hasattr(self.controller, "press_key"):
|
|
# Pass unrecognized keycodes through verbatim (KEYCODE_* names, numeric
|
|
# codes): the driver resolves or forwards them, matching the historical
|
|
# adb_server behavior of accepting any Android key event.
|
|
res = await self.controller.press_key(key)
|
|
err = getattr(res, "error", None) if res else None
|
|
if err:
|
|
return ActionResult.failure(
|
|
"press_key",
|
|
f"Error executing key press '{key}': {err}",
|
|
detail=str(err),
|
|
)
|
|
if not res:
|
|
return ActionResult.failure(
|
|
"press_key", f"Error executing key press '{key}'."
|
|
)
|
|
|
|
return ActionResult.success("press_key", f"Executed key press '{key}'.")
|
|
|
|
async def manage_app(self, action: str, app_name: str) -> ActionResult:
|
|
# Imported lazily: launch_app pulls in tool wrappers that are costly at import.
|
|
from artemis.tools.mobile.launch_app import find_package
|
|
from artemis.utils.app_launch_utils import launch_app_with_retries
|
|
|
|
res = find_package(self.ctx, app_name, use_fallback=False)
|
|
pkg = await res if inspect.iscoroutine(res) else res
|
|
if not pkg:
|
|
return ActionResult.failure(
|
|
"manage_app",
|
|
f"Error finding package for app: {app_name}",
|
|
code=ActionCode.PACKAGE_NOT_FOUND,
|
|
)
|
|
target_pkg = pkg
|
|
|
|
if action.lower() == "launch":
|
|
res_launch = launch_app_with_retries(self.ctx, target_pkg)
|
|
if inspect.iscoroutine(res_launch):
|
|
res_launch = await res_launch
|
|
if isinstance(res_launch, tuple):
|
|
success, error_msg = res_launch
|
|
else:
|
|
success, error_msg = bool(res_launch), ""
|
|
|
|
if success:
|
|
return ActionResult.success(
|
|
"manage_app",
|
|
f"Launched app '{app_name}' ({target_pkg}) successfully.",
|
|
)
|
|
return ActionResult.failure(
|
|
"manage_app",
|
|
f"Failed to launch app '{app_name}': {error_msg}",
|
|
detail=error_msg or None,
|
|
)
|
|
elif action.lower() == "stop":
|
|
if hasattr(self.controller, "terminate_app"):
|
|
res_term = self.controller.terminate_app(target_pkg)
|
|
if inspect.iscoroutine(res_term):
|
|
await res_term
|
|
return ActionResult.success(
|
|
"manage_app", f"Terminated app '{app_name}' successfully."
|
|
)
|
|
return ActionResult.failure(
|
|
"manage_app",
|
|
f"Invalid manage_app action: {action}",
|
|
code=ActionCode.INVALID_ARGS,
|
|
)
|
|
|
|
async def wait_for_delay(self, time_in_ms: int) -> ActionResult:
|
|
delay_s = max(0.0, float(time_in_ms) / 1000.0)
|
|
await asyncio.sleep(delay_s)
|
|
return ActionResult.success(
|
|
"wait_for_delay",
|
|
f"Waited for {time_in_ms}ms successfully.",
|
|
duration_ms=int(time_in_ms),
|
|
)
|
|
|
|
async def wait_for_text(
|
|
self, text: str, wait_state: str | None = None, timeout_ms: int | None = None
|
|
) -> ActionResult:
|
|
timeout_s = (timeout_ms or 5000) / 1000.0
|
|
start = time.time()
|
|
found = False
|
|
target_state = (wait_state or "appear").lower()
|
|
|
|
while time.time() - start < timeout_s:
|
|
tree = await self.controller.get_ui_elements()
|
|
text_present = text.lower() in str(tree or "").lower()
|
|
if target_state == "appear" and text_present:
|
|
found = True
|
|
break
|
|
elif target_state == "disappear" and not text_present:
|
|
found = True
|
|
break
|
|
await asyncio.sleep(0.5)
|
|
|
|
if found:
|
|
return ActionResult.success(
|
|
"wait_for_text",
|
|
f"Successfully waited for text '{text}' to {target_state}.",
|
|
)
|
|
return ActionResult.failure(
|
|
"wait_for_text",
|
|
f"Timed out waiting for text '{text}' to {target_state}.",
|
|
code=ActionCode.TIMEOUT,
|
|
)
|
|
|
|
async def open_link(self, url: str) -> ActionResult:
|
|
success = await self.controller.open_url(url)
|
|
if success:
|
|
return ActionResult.success("open_link", f"Opened link '{url}' successfully.")
|
|
return ActionResult.failure("open_link", f"Failed to open link '{url}'.")
|
|
|
|
async def erase_one_char(self) -> ActionResult:
|
|
success = await self.controller.erase_text(nb_chars=1)
|
|
if success:
|
|
return ActionResult.success("erase_one_char", "Erased one character.")
|
|
return ActionResult.failure("erase_one_char", "Failed to erase one character.")
|
|
|
|
async def focus_and_clear_text(self, nx: int, ny: int) -> ActionResult:
|
|
x, y = self._to_px(nx, ny)
|
|
err = await ensure_focus_at_coords(self.controller, x, y)
|
|
if err:
|
|
return ActionResult.failure(
|
|
"focus_and_clear_text",
|
|
f"Error focusing element: {err}",
|
|
detail=str(err),
|
|
normalized_coordinates=[int(nx), int(ny)],
|
|
)
|
|
success = await self.controller.erase_text()
|
|
if success:
|
|
return ActionResult.success(
|
|
"focus_and_clear_text",
|
|
f"Cleared text at [{nx}, {ny}] (normalized) successfully.",
|
|
normalized_coordinates=[int(nx), int(ny)],
|
|
)
|
|
return ActionResult.failure("focus_and_clear_text", "Failed to erase text.")
|
|
|
|
# --- Internal observation primitives ---------------------------------------------
|
|
|
|
async def take_screenshot(self) -> str:
|
|
return await self.controller.take_screenshot()
|
|
|
|
async def get_ui_elements(self) -> Any:
|
|
return await self.controller.get_ui_elements()
|
|
|
|
async def get_screen_data(self) -> Any:
|
|
return await self.controller.get_screen_data()
|