somew1nd
a45daf23f8
Report device actions as dispatched and preserve full failure details. Route Flash through McpActionExecutor and use ToolFailure across helper tools and agent loops instead of inferring failure from response text. Require model-provided descriptions for coordinate targets. Keep observed element metadata separate from those descriptions in safety checks, visual summaries, history, and frontend labels. Add Gemini Interactions API video analysis with configurable processing mode, chunk size, and timeouts. Centralize Google provider capabilities and token usage normalization, and update google-genai to 2.22.0. Preserve system instructions on follow-up interactions, replan pending chunks when falling back to static processing, and reject unanswered native video and audio segments instead of caching placeholder results. Keep helper-tool failure status intact through history tools and committee calls. Validation: 1,762 Python unit tests passed (3 skipped), 40 frontend formatter tests passed, and the production UI build completed. Ruff lint, formatting, the quality ratchet, and git diff checks passed.
415 lines
16 KiB
Python
415 lines
16 KiB
Python
# Copyright 2026 Google LLC
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
"""ADB-backed actuator: the reference implementation of the actuator contract.
|
|
|
|
Outcome messages describe the dispatched command, such as
|
|
"Tapped at [x, y] (normalized)." The agent verifies the effect from subsequent
|
|
observations.
|
|
|
|
Error convention: device-level refusals the controller *reports* (an ``error`` field,
|
|
a falsy success flag) come back as ``ActionResult(ok=False)`` with the historical
|
|
message. Unexpected exceptions are allowed to propagate -- the caller (executor or MCP
|
|
server layer) owns the ``"Error during X: {e}"`` wrapping, exactly as the original
|
|
executor's ``try/except`` did.
|
|
"""
|
|
|
|
import asyncio
|
|
import inspect
|
|
import re
|
|
import time
|
|
from typing import Any
|
|
|
|
from artemis.context import ArtemisContext
|
|
from artemis.controllers.unified_controller import UnifiedMobileController
|
|
from artemis.mcp.action_manifest import DEVICE_ACTIONS, ExtensionTool
|
|
from artemis.mcp.action_types import ActionCode, ActionResult
|
|
from artemis.utils.logger import get_logger
|
|
|
|
logger = get_logger(__name__)
|
|
|
|
__all__ = ["AdbActuator", "find_element_at_coords", "ensure_focus_at_coords"]
|
|
|
|
|
|
def find_element_at_coords(elements: list[dict], x: int, y: int) -> dict | None:
|
|
"""Finds the smallest (leaf-most) focusable element containing pixel [x, y].
|
|
|
|
Moved from ``artemis.mcp.adb_server`` so both the stdio server and in-process
|
|
actuator share one implementation.
|
|
"""
|
|
matching_element = None
|
|
min_area = float("inf")
|
|
|
|
for elem in elements:
|
|
is_focusable = (
|
|
elem.get("focusable") == "true"
|
|
or elem.get("clickable") == "true"
|
|
or "EditText" in str(elem.get("class", ""))
|
|
)
|
|
if not is_focusable:
|
|
continue
|
|
|
|
bounds_str = elem.get("bounds")
|
|
if bounds_str and isinstance(bounds_str, str):
|
|
match = re.match(r"\[(\-?\d+),(\-?\d+)\]\[(\-?\d+),(\-?\d+)\]", bounds_str)
|
|
if match:
|
|
x1, y1, x2, y2 = map(int, match.groups())
|
|
if x1 <= x <= x2 and y1 <= y <= y2:
|
|
area = (x2 - x1) * (y2 - y1)
|
|
if area < min_area:
|
|
min_area = area
|
|
matching_element = elem
|
|
|
|
return matching_element
|
|
|
|
|
|
async def ensure_focus_at_coords(controller, x: int, y: int) -> str | None:
|
|
"""Ensures the element at pixel [x, y] is focused, tapping only if needed.
|
|
|
|
Returns an error string on failure, ``None`` on success. Moved from
|
|
``artemis.mcp.adb_server``.
|
|
"""
|
|
try:
|
|
elements = await controller.get_ui_elements()
|
|
elem = find_element_at_coords(elements, x, y)
|
|
if elem and elem.get("focused") == "true":
|
|
logger.info(f"Element under [{x}, {y}] is already focused. Skipping tap.")
|
|
return None
|
|
except Exception as e:
|
|
logger.warning(f"Failed to check focus status: {e}. Falling back to unconditional tap.")
|
|
|
|
result = await controller.tap_at(x=x, y=y)
|
|
if hasattr(result, "error") and result.error:
|
|
return result.error
|
|
|
|
# Wait for the UI to settle and keyboard to pop up after tapping
|
|
await asyncio.sleep(1.0)
|
|
return None
|
|
|
|
|
|
class AdbActuator:
|
|
"""Drives an Android device through ``UnifiedMobileController`` over ADB."""
|
|
|
|
def __init__(
|
|
self,
|
|
ctx: ArtemisContext,
|
|
controller: UnifiedMobileController | None = None,
|
|
):
|
|
self.ctx = ctx
|
|
self.controller = controller or UnifiedMobileController(ctx)
|
|
|
|
# --- Contract --------------------------------------------------------------------
|
|
|
|
def capabilities(self) -> frozenset[str]:
|
|
return DEVICE_ACTIONS
|
|
|
|
def extensions(self) -> list[ExtensionTool]:
|
|
return []
|
|
|
|
# --- Coordinate helpers ----------------------------------------------------------
|
|
|
|
def _dims(self) -> tuple[int, int]:
|
|
width = getattr(self.ctx.device, "device_width", 1080) if self.ctx.device else 1080
|
|
height = getattr(self.ctx.device, "device_height", 2400) if self.ctx.device else 2400
|
|
return width or 1080, height or 2400
|
|
|
|
def _to_px(self, nx: int, ny: int) -> tuple[int, int]:
|
|
width, height = self._dims()
|
|
x = int(max(0, min(width - 1, int(nx) * width / 1000)))
|
|
y = int(max(0, min(height - 1, int(ny) * height / 1000)))
|
|
return x, y
|
|
|
|
# --- Required action -------------------------------------------------------------
|
|
|
|
async def click_sequence(
|
|
self, points: list[tuple[int, int]], delay_ms: int = 50
|
|
) -> ActionResult:
|
|
outcomes = []
|
|
for i, (nx, ny) in enumerate(points):
|
|
x, y = self._to_px(nx, ny)
|
|
result = await self.controller.tap_at(x, y)
|
|
err = getattr(result, "error", None) if result else None
|
|
if err:
|
|
return ActionResult.failure(
|
|
"click_sequence",
|
|
f"Error executing click at step {i + 1}: {err}",
|
|
detail=str(err),
|
|
)
|
|
outcomes.append(f"[{nx}, {ny}]")
|
|
if i < len(points) - 1:
|
|
await asyncio.sleep(max(0, delay_ms) / 1000.0)
|
|
|
|
return ActionResult.success(
|
|
"click_sequence",
|
|
f"Tapped in sequence at {'; '.join(outcomes)} (normalized).",
|
|
)
|
|
|
|
# --- Optional actions ------------------------------------------------------------
|
|
|
|
async def click(self, nx: int, ny: int, times: int = 1, delay_ms: int = 100) -> ActionResult:
|
|
x, y = self._to_px(nx, ny)
|
|
result = await self.controller.tap_at(x, y, times=times, delay_ms=delay_ms)
|
|
err = getattr(result, "error", None) if result else None
|
|
if err is not None:
|
|
return ActionResult.failure("click", f"Error executing click: {err}", detail=str(err))
|
|
return ActionResult.success(
|
|
"click",
|
|
f"Tapped at [{nx}, {ny}] (normalized).",
|
|
normalized_coordinates=[int(nx), int(ny)],
|
|
)
|
|
|
|
async def long_press(self, nx: int, ny: int, duration_ms: int = 1000) -> ActionResult:
|
|
x, y = self._to_px(nx, ny)
|
|
result = await self.controller.tap_at(
|
|
x, y, long_press=True, long_press_duration=duration_ms
|
|
)
|
|
err = getattr(result, "error", None) if result else None
|
|
if err is not None:
|
|
return ActionResult.failure(
|
|
"long_press", f"Error executing long press: {err}", detail=str(err)
|
|
)
|
|
return ActionResult.success(
|
|
"long_press",
|
|
f"Long-pressed at [{nx}, {ny}] (normalized) for {duration_ms}ms.",
|
|
normalized_coordinates=[int(nx), int(ny)],
|
|
duration_ms=duration_ms,
|
|
)
|
|
|
|
async def input_text(
|
|
self,
|
|
text: str,
|
|
target: tuple[int, int] | None = None,
|
|
clear_exist: bool = True,
|
|
) -> ActionResult:
|
|
coords = None
|
|
if target:
|
|
nx, ny = target
|
|
x, y = self._to_px(nx, ny)
|
|
coords = [int(nx), int(ny)]
|
|
err = await ensure_focus_at_coords(self.controller, x, y)
|
|
if err:
|
|
return ActionResult.failure(
|
|
"input_text",
|
|
f"Error focusing element: {err}",
|
|
detail=str(err),
|
|
normalized_coordinates=coords,
|
|
)
|
|
if clear_exist:
|
|
if hasattr(self.controller, "erase_text") and not await self.controller.erase_text():
|
|
return ActionResult.failure(
|
|
"input_text",
|
|
"Failed to clear existing text",
|
|
normalized_coordinates=coords,
|
|
)
|
|
elif hasattr(self.controller, "press_key"):
|
|
# KEYCODE_MOVE_END: move cursor to the end for reliable append.
|
|
await self.controller.press_key("123")
|
|
if hasattr(self.controller, "type_text") and not await self.controller.type_text(
|
|
text, clear_existing=False
|
|
):
|
|
return ActionResult.failure(
|
|
"input_text",
|
|
f"Failed typing '{text}'.",
|
|
normalized_coordinates=coords,
|
|
)
|
|
|
|
return ActionResult.success(
|
|
"input_text",
|
|
f"Typed '{text}'"
|
|
+ (f" at [{coords[0]}, {coords[1]}] (normalized)." if coords else "."),
|
|
normalized_coordinates=coords,
|
|
)
|
|
|
|
async def swipe(
|
|
self,
|
|
start: tuple[int, int],
|
|
end: tuple[int, int],
|
|
duration_ms: int = 800,
|
|
) -> ActionResult:
|
|
nx1, ny1 = start
|
|
nx2, ny2 = end
|
|
px1, py1 = self._to_px(nx1, ny1)
|
|
px2, py2 = self._to_px(nx2, ny2)
|
|
err = await self.controller.swipe_coords(px1, py1, px2, py2, duration_ms)
|
|
if err:
|
|
return ActionResult.failure("swipe", f"Error dragging: {err}", detail=str(err))
|
|
return ActionResult.success(
|
|
"swipe",
|
|
f"Swiped from [{nx1}, {ny1}] to [{nx2}, {ny2}] (normalized).",
|
|
normalized_coordinates=[int(nx1), int(ny1), int(nx2), int(ny2)],
|
|
duration_ms=duration_ms,
|
|
)
|
|
|
|
async def press_key(self, key: str) -> ActionResult:
|
|
key_str = str(key).lower()
|
|
if not key_str:
|
|
return ActionResult.failure(
|
|
"press_key",
|
|
f"Error executing key press '{key}'.",
|
|
code=ActionCode.INVALID_ARGS,
|
|
)
|
|
|
|
if key_str == "back" and hasattr(self.controller, "go_back"):
|
|
await self.controller.go_back()
|
|
elif key_str == "home" and hasattr(self.controller, "go_home"):
|
|
await self.controller.go_home()
|
|
elif key_str == "enter" and hasattr(self.controller, "press_enter"):
|
|
await self.controller.press_enter()
|
|
elif hasattr(self.controller, "press_key"):
|
|
# Pass unrecognized keycodes through verbatim (KEYCODE_* names, numeric
|
|
# codes): the driver resolves or forwards them, matching the historical
|
|
# adb_server behavior of accepting any Android key event.
|
|
res = await self.controller.press_key(key)
|
|
err = getattr(res, "error", None) if res else None
|
|
if err:
|
|
return ActionResult.failure(
|
|
"press_key",
|
|
f"Error executing key press '{key}': {err}",
|
|
detail=str(err),
|
|
)
|
|
if not res:
|
|
return ActionResult.failure("press_key", f"Error executing key press '{key}'.")
|
|
|
|
return ActionResult.success("press_key", f"Pressed key '{key}'.")
|
|
|
|
async def manage_app(self, action: str, app_name: str) -> ActionResult:
|
|
# Imported lazily: launch_app pulls in tool wrappers that are costly at import.
|
|
from artemis.tools.mobile.launch_app import find_package
|
|
from artemis.utils.app_launch_utils import launch_app_with_retries
|
|
|
|
res = find_package(self.ctx, app_name, use_fallback=False)
|
|
pkg = await res if inspect.iscoroutine(res) else res
|
|
if not pkg:
|
|
return ActionResult.failure(
|
|
"manage_app",
|
|
f"Error finding package for app: {app_name}",
|
|
code=ActionCode.PACKAGE_NOT_FOUND,
|
|
)
|
|
target_pkg = pkg
|
|
|
|
if action.lower() == "launch":
|
|
res_launch = launch_app_with_retries(self.ctx, target_pkg)
|
|
if inspect.iscoroutine(res_launch):
|
|
res_launch = await res_launch
|
|
if isinstance(res_launch, tuple):
|
|
success, error_msg = res_launch
|
|
else:
|
|
success, error_msg = bool(res_launch), ""
|
|
|
|
if success:
|
|
return ActionResult.success(
|
|
"manage_app",
|
|
f"Launched app '{app_name}' ({target_pkg}); foreground confirmed.",
|
|
)
|
|
return ActionResult.failure(
|
|
"manage_app",
|
|
f"Failed to launch app '{app_name}': {error_msg}",
|
|
detail=error_msg or None,
|
|
)
|
|
elif action.lower() == "stop":
|
|
if hasattr(self.controller, "terminate_app"):
|
|
res_term = self.controller.terminate_app(target_pkg)
|
|
if inspect.iscoroutine(res_term):
|
|
await res_term
|
|
return ActionResult.success(
|
|
"manage_app", f"Force-stopped app '{app_name}' ({target_pkg})."
|
|
)
|
|
return ActionResult.failure(
|
|
"manage_app",
|
|
f"Invalid manage_app action: {action}",
|
|
code=ActionCode.INVALID_ARGS,
|
|
)
|
|
|
|
async def wait_for_delay(self, time_in_ms: int) -> ActionResult:
|
|
delay_s = max(0.0, float(time_in_ms) / 1000.0)
|
|
await asyncio.sleep(delay_s)
|
|
return ActionResult.success(
|
|
"wait_for_delay",
|
|
f"Waited {time_in_ms}ms.",
|
|
duration_ms=int(time_in_ms),
|
|
)
|
|
|
|
async def wait_for_text(
|
|
self, text: str, wait_state: str | None = None, timeout_ms: int | None = None
|
|
) -> ActionResult:
|
|
timeout_s = (timeout_ms or 5000) / 1000.0
|
|
start = time.time()
|
|
found = False
|
|
target_state = (wait_state or "appear").lower()
|
|
|
|
while time.time() - start < timeout_s:
|
|
tree = await self.controller.get_ui_elements()
|
|
text_present = text.lower() in str(tree or "").lower()
|
|
if target_state == "appear" and text_present:
|
|
found = True
|
|
break
|
|
elif target_state == "disappear" and not text_present:
|
|
found = True
|
|
break
|
|
await asyncio.sleep(0.5)
|
|
|
|
if found:
|
|
return ActionResult.success(
|
|
"wait_for_text",
|
|
f"Text '{text}' {'appeared' if target_state == 'appear' else 'disappeared'}"
|
|
f" in the UI tree after {int((time.time() - start) * 1000)}ms.",
|
|
)
|
|
return ActionResult.failure(
|
|
"wait_for_text",
|
|
f"Timed out waiting for text '{text}' to {target_state}.",
|
|
code=ActionCode.TIMEOUT,
|
|
)
|
|
|
|
async def open_link(self, url: str) -> ActionResult:
|
|
success = await self.controller.open_url(url)
|
|
if success:
|
|
return ActionResult.success("open_link", f"Opened link '{url}'.")
|
|
return ActionResult.failure("open_link", f"Failed to open link '{url}'.")
|
|
|
|
async def erase_one_char(self) -> ActionResult:
|
|
success = await self.controller.erase_text(nb_chars=1)
|
|
if success:
|
|
return ActionResult.success("erase_one_char", "Erased one character.")
|
|
return ActionResult.failure("erase_one_char", "Failed to erase one character.")
|
|
|
|
async def focus_and_clear_text(self, nx: int, ny: int) -> ActionResult:
|
|
x, y = self._to_px(nx, ny)
|
|
err = await ensure_focus_at_coords(self.controller, x, y)
|
|
if err:
|
|
return ActionResult.failure(
|
|
"focus_and_clear_text",
|
|
f"Error focusing element: {err}",
|
|
detail=str(err),
|
|
normalized_coordinates=[int(nx), int(ny)],
|
|
)
|
|
success = await self.controller.erase_text()
|
|
if success:
|
|
return ActionResult.success(
|
|
"focus_and_clear_text",
|
|
f"Cleared text at [{nx}, {ny}] (normalized).",
|
|
normalized_coordinates=[int(nx), int(ny)],
|
|
)
|
|
return ActionResult.failure("focus_and_clear_text", "Failed to erase text.")
|
|
|
|
# --- Internal observation primitives ---------------------------------------------
|
|
|
|
async def take_screenshot(self) -> str:
|
|
return await self.controller.take_screenshot()
|
|
|
|
async def get_ui_elements(self) -> Any:
|
|
return await self.controller.get_ui_elements()
|
|
|
|
async def get_screen_data(self) -> Any:
|
|
return await self.controller.get_screen_data()
|