somew1nd
67248c887e
Add execution incidents, action bursts and shared Flash history. Expose verification and Explorer settings with streamed check results. Support graceful cancellation, recording recovery and bundled UI resources. Add CI quality checks and clean up comments and documentation.
412 lines
16 KiB
Python
412 lines
16 KiB
Python
# Copyright 2026 Google LLC
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""ADB-backed actuator: the reference implementation of the actuator contract.
|
|
|
|
The device-call bodies here were extracted from
|
|
``artemis.agents.validator.tool_declarations.MobileActionExecutor``; the outcome
|
|
message wording is preserved byte-for-byte because agent transcripts, trace payloads,
|
|
and existing tests assert on it.
|
|
|
|
Error convention: device-level refusals the controller *reports* (an ``error`` field,
|
|
a falsy success flag) come back as ``ActionResult(ok=False)`` with the historical
|
|
message. Unexpected exceptions are allowed to propagate -- the caller (executor or MCP
|
|
server layer) owns the ``"Error during X: {e}"`` wrapping, exactly as the original
|
|
executor's ``try/except`` did.
|
|
"""
|
|
|
|
import asyncio
|
|
import inspect
|
|
import re
|
|
import time
|
|
from typing import Any
|
|
|
|
from artemis.context import ArtemisContext
|
|
from artemis.controllers.unified_controller import UnifiedMobileController
|
|
from artemis.mcp.action_manifest import DEVICE_ACTIONS, ExtensionTool
|
|
from artemis.mcp.action_types import ActionCode, ActionResult
|
|
from artemis.utils.logger import get_logger
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
__all__ = ["AdbActuator", "find_element_at_coords", "ensure_focus_at_coords"]
|
|
|
|
|
|
def find_element_at_coords(elements: list[dict], x: int, y: int) -> dict | None:
|
|
"""Finds the smallest (leaf-most) focusable element containing pixel [x, y].
|
|
|
|
Moved from ``artemis.mcp.adb_server`` so both the stdio server and in-process
|
|
actuator share one implementation.
|
|
"""
|
|
matching_element = None
|
|
min_area = float("inf")
|
|
|
|
for elem in elements:
|
|
is_focusable = (
|
|
elem.get("focusable") == "true"
|
|
or elem.get("clickable") == "true"
|
|
or "EditText" in str(elem.get("class", ""))
|
|
)
|
|
if not is_focusable:
|
|
continue
|
|
|
|
bounds_str = elem.get("bounds")
|
|
if bounds_str and isinstance(bounds_str, str):
|
|
match = re.match(r"\[(\-?\d+),(\-?\d+)\]\[(\-?\d+),(\-?\d+)\]", bounds_str)
|
|
if match:
|
|
x1, y1, x2, y2 = map(int, match.groups())
|
|
if x1 <= x <= x2 and y1 <= y <= y2:
|
|
area = (x2 - x1) * (y2 - y1)
|
|
if area < min_area:
|
|
min_area = area
|
|
matching_element = elem
|
|
|
|
return matching_element
|
|
|
|
|
|
async def ensure_focus_at_coords(controller, x: int, y: int) -> str | None:
|
|
"""Ensures the element at pixel [x, y] is focused, tapping only if needed.
|
|
|
|
Returns an error string on failure, ``None`` on success. Moved from
|
|
``artemis.mcp.adb_server``.
|
|
"""
|
|
try:
|
|
elements = await controller.get_ui_elements()
|
|
elem = find_element_at_coords(elements, x, y)
|
|
if elem and elem.get("focused") == "true":
|
|
logger.info(f"Element under [{x}, {y}] is already focused. Skipping tap.")
|
|
return None
|
|
except Exception as e:
|
|
logger.warning(f"Failed to check focus status: {e}. Falling back to unconditional tap.")
|
|
|
|
result = await controller.tap_at(x=x, y=y)
|
|
if hasattr(result, "error") and result.error:
|
|
return result.error
|
|
|
|
# Wait for the UI to settle and keyboard to pop up after tapping
|
|
await asyncio.sleep(1.0)
|
|
return None
|
|
|
|
|
|
class AdbActuator:
|
|
"""Drives an Android device through ``UnifiedMobileController`` over ADB."""
|
|
|
|
def __init__(
|
|
self,
|
|
ctx: ArtemisContext,
|
|
controller: UnifiedMobileController | None = None,
|
|
):
|
|
self.ctx = ctx
|
|
self.controller = controller or UnifiedMobileController(ctx)
|
|
|
|
# --- Contract --------------------------------------------------------------------
|
|
|
|
def capabilities(self) -> frozenset[str]:
|
|
return DEVICE_ACTIONS
|
|
|
|
def extensions(self) -> list[ExtensionTool]:
|
|
return []
|
|
|
|
# --- Coordinate helpers ----------------------------------------------------------
|
|
|
|
def _dims(self) -> tuple[int, int]:
|
|
width = getattr(self.ctx.device, "device_width", 1080) if self.ctx.device else 1080
|
|
height = getattr(self.ctx.device, "device_height", 2400) if self.ctx.device else 2400
|
|
return width or 1080, height or 2400
|
|
|
|
def _to_px(self, nx: int, ny: int) -> tuple[int, int]:
|
|
width, height = self._dims()
|
|
x = int(max(0, min(width - 1, int(nx) * width / 1000)))
|
|
y = int(max(0, min(height - 1, int(ny) * height / 1000)))
|
|
return x, y
|
|
|
|
# --- Required action -------------------------------------------------------------
|
|
|
|
async def click_sequence(
|
|
self, points: list[tuple[int, int]], delay_ms: int = 50
|
|
) -> ActionResult:
|
|
outcomes = []
|
|
for i, (nx, ny) in enumerate(points):
|
|
x, y = self._to_px(nx, ny)
|
|
result = await self.controller.tap_at(x, y)
|
|
err = getattr(result, "error", None) if result else None
|
|
if err:
|
|
return ActionResult.failure(
|
|
"click_sequence",
|
|
f"Error executing click at step {i + 1}: {err}",
|
|
detail=str(err),
|
|
)
|
|
outcomes.append(f"Tapped at [{x}, {y}]")
|
|
if i < len(points) - 1:
|
|
await asyncio.sleep(max(0, delay_ms) / 1000.0)
|
|
|
|
return ActionResult.success(
|
|
"click_sequence",
|
|
f"Sequence clicked successfully: {'; '.join(outcomes)}",
|
|
)
|
|
|
|
# --- Optional actions ------------------------------------------------------------
|
|
|
|
async def click(self, nx: int, ny: int, times: int = 1, delay_ms: int = 100) -> ActionResult:
|
|
x, y = self._to_px(nx, ny)
|
|
result = await self.controller.tap_at(x, y, times=times, delay_ms=delay_ms)
|
|
err = getattr(result, "error", None) if result else None
|
|
if err is not None:
|
|
return ActionResult.failure("click", f"Error executing click: {err}", detail=str(err))
|
|
return ActionResult.success(
|
|
"click",
|
|
f"Clicked at [{nx}, {ny}] (normalized) successfully.",
|
|
normalized_coordinates=[int(nx), int(ny)],
|
|
)
|
|
|
|
async def long_press(self, nx: int, ny: int, duration_ms: int = 1000) -> ActionResult:
|
|
x, y = self._to_px(nx, ny)
|
|
result = await self.controller.tap_at(
|
|
x, y, long_press=True, long_press_duration=duration_ms
|
|
)
|
|
err = getattr(result, "error", None) if result else None
|
|
if err is not None:
|
|
return ActionResult.failure(
|
|
"long_press", f"Error executing long press: {err}", detail=str(err)
|
|
)
|
|
return ActionResult.success(
|
|
"long_press",
|
|
f"Long pressed at [{nx}, {ny}] (normalized) for {duration_ms}ms successfully.",
|
|
normalized_coordinates=[int(nx), int(ny)],
|
|
duration_ms=duration_ms,
|
|
)
|
|
|
|
async def input_text(
|
|
self,
|
|
text: str,
|
|
target: tuple[int, int] | None = None,
|
|
clear_exist: bool = True,
|
|
) -> ActionResult:
|
|
coords = None
|
|
if target:
|
|
nx, ny = target
|
|
x, y = self._to_px(nx, ny)
|
|
coords = [int(nx), int(ny)]
|
|
err = await ensure_focus_at_coords(self.controller, x, y)
|
|
if err:
|
|
return ActionResult.failure(
|
|
"input_text",
|
|
f"Error focusing element: {err}",
|
|
detail=str(err),
|
|
normalized_coordinates=coords,
|
|
)
|
|
if clear_exist:
|
|
if hasattr(self.controller, "erase_text") and not await self.controller.erase_text():
|
|
return ActionResult.failure(
|
|
"input_text",
|
|
"Failed to clear existing text",
|
|
normalized_coordinates=coords,
|
|
)
|
|
elif hasattr(self.controller, "press_key"):
|
|
# KEYCODE_MOVE_END: move cursor to the end for reliable append.
|
|
await self.controller.press_key("123")
|
|
if hasattr(self.controller, "type_text") and not await self.controller.type_text(
|
|
text, clear_existing=False
|
|
):
|
|
return ActionResult.failure(
|
|
"input_text",
|
|
f"Failed typing '{text}'.",
|
|
normalized_coordinates=coords,
|
|
)
|
|
|
|
return ActionResult.success(
|
|
"input_text",
|
|
f"Executed typing '{text}'.",
|
|
normalized_coordinates=coords,
|
|
)
|
|
|
|
async def swipe(
|
|
self,
|
|
start: tuple[int, int],
|
|
end: tuple[int, int],
|
|
duration_ms: int = 800,
|
|
) -> ActionResult:
|
|
nx1, ny1 = start
|
|
nx2, ny2 = end
|
|
px1, py1 = self._to_px(nx1, ny1)
|
|
px2, py2 = self._to_px(nx2, ny2)
|
|
err = await self.controller.swipe_coords(px1, py1, px2, py2, duration_ms)
|
|
if err:
|
|
return ActionResult.failure("swipe", f"Error dragging: {err}", detail=str(err))
|
|
return ActionResult.success(
|
|
"swipe",
|
|
f"Swipe completed successfully. Swiped from [{nx1}, {ny1}] to [{nx2}, {ny2}].",
|
|
normalized_coordinates=[int(nx1), int(ny1), int(nx2), int(ny2)],
|
|
duration_ms=duration_ms,
|
|
)
|
|
|
|
async def press_key(self, key: str) -> ActionResult:
|
|
key_str = str(key).lower()
|
|
if not key_str:
|
|
return ActionResult.failure(
|
|
"press_key",
|
|
f"Error executing key press '{key}'.",
|
|
code=ActionCode.INVALID_ARGS,
|
|
)
|
|
|
|
if key_str == "back" and hasattr(self.controller, "go_back"):
|
|
await self.controller.go_back()
|
|
elif key_str == "home" and hasattr(self.controller, "go_home"):
|
|
await self.controller.go_home()
|
|
elif key_str == "enter" and hasattr(self.controller, "press_enter"):
|
|
await self.controller.press_enter()
|
|
elif hasattr(self.controller, "press_key"):
|
|
# Pass unrecognized keycodes through verbatim (KEYCODE_* names, numeric
|
|
# codes): the driver resolves or forwards them, matching the historical
|
|
# adb_server behavior of accepting any Android key event.
|
|
res = await self.controller.press_key(key)
|
|
err = getattr(res, "error", None) if res else None
|
|
if err:
|
|
return ActionResult.failure(
|
|
"press_key",
|
|
f"Error executing key press '{key}': {err}",
|
|
detail=str(err),
|
|
)
|
|
if not res:
|
|
return ActionResult.failure("press_key", f"Error executing key press '{key}'.")
|
|
|
|
return ActionResult.success("press_key", f"Executed key press '{key}'.")
|
|
|
|
async def manage_app(self, action: str, app_name: str) -> ActionResult:
|
|
# Imported lazily: launch_app pulls in tool wrappers that are costly at import.
|
|
from artemis.tools.mobile.launch_app import find_package
|
|
from artemis.utils.app_launch_utils import launch_app_with_retries
|
|
|
|
res = find_package(self.ctx, app_name, use_fallback=False)
|
|
pkg = await res if inspect.iscoroutine(res) else res
|
|
if not pkg:
|
|
return ActionResult.failure(
|
|
"manage_app",
|
|
f"Error finding package for app: {app_name}",
|
|
code=ActionCode.PACKAGE_NOT_FOUND,
|
|
)
|
|
target_pkg = pkg
|
|
|
|
if action.lower() == "launch":
|
|
res_launch = launch_app_with_retries(self.ctx, target_pkg)
|
|
if inspect.iscoroutine(res_launch):
|
|
res_launch = await res_launch
|
|
if isinstance(res_launch, tuple):
|
|
success, error_msg = res_launch
|
|
else:
|
|
success, error_msg = bool(res_launch), ""
|
|
|
|
if success:
|
|
return ActionResult.success(
|
|
"manage_app",
|
|
f"Launched app '{app_name}' ({target_pkg}) successfully.",
|
|
)
|
|
return ActionResult.failure(
|
|
"manage_app",
|
|
f"Failed to launch app '{app_name}': {error_msg}",
|
|
detail=error_msg or None,
|
|
)
|
|
elif action.lower() == "stop":
|
|
if hasattr(self.controller, "terminate_app"):
|
|
res_term = self.controller.terminate_app(target_pkg)
|
|
if inspect.iscoroutine(res_term):
|
|
await res_term
|
|
return ActionResult.success("manage_app", f"Terminated app '{app_name}' successfully.")
|
|
return ActionResult.failure(
|
|
"manage_app",
|
|
f"Invalid manage_app action: {action}",
|
|
code=ActionCode.INVALID_ARGS,
|
|
)
|
|
|
|
async def wait_for_delay(self, time_in_ms: int) -> ActionResult:
|
|
delay_s = max(0.0, float(time_in_ms) / 1000.0)
|
|
await asyncio.sleep(delay_s)
|
|
return ActionResult.success(
|
|
"wait_for_delay",
|
|
f"Waited for {time_in_ms}ms successfully.",
|
|
duration_ms=int(time_in_ms),
|
|
)
|
|
|
|
async def wait_for_text(
|
|
self, text: str, wait_state: str | None = None, timeout_ms: int | None = None
|
|
) -> ActionResult:
|
|
timeout_s = (timeout_ms or 5000) / 1000.0
|
|
start = time.time()
|
|
found = False
|
|
target_state = (wait_state or "appear").lower()
|
|
|
|
while time.time() - start < timeout_s:
|
|
tree = await self.controller.get_ui_elements()
|
|
text_present = text.lower() in str(tree or "").lower()
|
|
if target_state == "appear" and text_present:
|
|
found = True
|
|
break
|
|
elif target_state == "disappear" and not text_present:
|
|
found = True
|
|
break
|
|
await asyncio.sleep(0.5)
|
|
|
|
if found:
|
|
return ActionResult.success(
|
|
"wait_for_text",
|
|
f"Successfully waited for text '{text}' to {target_state}.",
|
|
)
|
|
return ActionResult.failure(
|
|
"wait_for_text",
|
|
f"Timed out waiting for text '{text}' to {target_state}.",
|
|
code=ActionCode.TIMEOUT,
|
|
)
|
|
|
|
async def open_link(self, url: str) -> ActionResult:
|
|
success = await self.controller.open_url(url)
|
|
if success:
|
|
return ActionResult.success("open_link", f"Opened link '{url}' successfully.")
|
|
return ActionResult.failure("open_link", f"Failed to open link '{url}'.")
|
|
|
|
async def erase_one_char(self) -> ActionResult:
|
|
success = await self.controller.erase_text(nb_chars=1)
|
|
if success:
|
|
return ActionResult.success("erase_one_char", "Erased one character.")
|
|
return ActionResult.failure("erase_one_char", "Failed to erase one character.")
|
|
|
|
async def focus_and_clear_text(self, nx: int, ny: int) -> ActionResult:
|
|
x, y = self._to_px(nx, ny)
|
|
err = await ensure_focus_at_coords(self.controller, x, y)
|
|
if err:
|
|
return ActionResult.failure(
|
|
"focus_and_clear_text",
|
|
f"Error focusing element: {err}",
|
|
detail=str(err),
|
|
normalized_coordinates=[int(nx), int(ny)],
|
|
)
|
|
success = await self.controller.erase_text()
|
|
if success:
|
|
return ActionResult.success(
|
|
"focus_and_clear_text",
|
|
f"Cleared text at [{nx}, {ny}] (normalized) successfully.",
|
|
normalized_coordinates=[int(nx), int(ny)],
|
|
)
|
|
return ActionResult.failure("focus_and_clear_text", "Failed to erase text.")
|
|
|
|
# --- Internal observation primitives ---------------------------------------------
|
|
|
|
async def take_screenshot(self) -> str:
|
|
return await self.controller.take_screenshot()
|
|
|
|
async def get_ui_elements(self) -> Any:
|
|
return await self.controller.get_ui_elements()
|
|
|
|
async def get_screen_data(self) -> Any:
|
|
return await self.controller.get_screen_data()
|