# Copyright 2026 Google LLC # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """ADB-backed actuator: the reference implementation of the actuator contract. Outcome messages describe the dispatched command, such as "Tapped at [x, y] (normalized)." The agent verifies the effect from subsequent observations. Error convention: device-level refusals the controller *reports* (an ``error`` field, a falsy success flag) come back as ``ActionResult(ok=False)`` with the historical message. Unexpected exceptions are allowed to propagate -- the caller (executor or MCP server layer) owns the ``"Error during X: {e}"`` wrapping, exactly as the original executor's ``try/except`` did. """ import asyncio import inspect import re import time from typing import Any from artemis.context import ArtemisContext from artemis.controllers.unified_controller import UnifiedMobileController from artemis.mcp.action_manifest import DEVICE_ACTIONS, ExtensionTool from artemis.mcp.action_types import ActionCode, ActionResult from artemis.utils.logger import get_logger logger = get_logger(__name__) __all__ = ["AdbActuator", "find_element_at_coords", "ensure_focus_at_coords"] def find_element_at_coords(elements: list[dict], x: int, y: int) -> dict | None: """Finds the smallest (leaf-most) focusable element containing pixel [x, y]. Moved from ``artemis.mcp.adb_server`` so both the stdio server and in-process actuator share one implementation. """ matching_element = None min_area = float("inf") for elem in elements: is_focusable = ( elem.get("focusable") == "true" or elem.get("clickable") == "true" or "EditText" in str(elem.get("class", "")) ) if not is_focusable: continue bounds_str = elem.get("bounds") if bounds_str and isinstance(bounds_str, str): match = re.match(r"\[(\-?\d+),(\-?\d+)\]\[(\-?\d+),(\-?\d+)\]", bounds_str) if match: x1, y1, x2, y2 = map(int, match.groups()) if x1 <= x <= x2 and y1 <= y <= y2: area = (x2 - x1) * (y2 - y1) if area < min_area: min_area = area matching_element = elem return matching_element async def ensure_focus_at_coords(controller, x: int, y: int) -> str | None: """Ensures the element at pixel [x, y] is focused, tapping only if needed. Returns an error string on failure, ``None`` on success. Moved from ``artemis.mcp.adb_server``. """ try: elements = await controller.get_ui_elements() elem = find_element_at_coords(elements, x, y) if elem and elem.get("focused") == "true": logger.info(f"Element under [{x}, {y}] is already focused. Skipping tap.") return None except Exception as e: logger.warning(f"Failed to check focus status: {e}. Falling back to unconditional tap.") result = await controller.tap_at(x=x, y=y) if hasattr(result, "error") and result.error: return result.error # Wait for the UI to settle and keyboard to pop up after tapping await asyncio.sleep(1.0) return None class AdbActuator: """Drives an Android device through ``UnifiedMobileController`` over ADB.""" def __init__( self, ctx: ArtemisContext, controller: UnifiedMobileController | None = None, ): self.ctx = ctx self.controller = controller or UnifiedMobileController(ctx) # --- Contract -------------------------------------------------------------------- def capabilities(self) -> frozenset[str]: from artemis.mcp.native_actions import NATIVE_ACTIONS, native_tools_for_capabilities native = self.controller.driver.native_capabilities return (DEVICE_ACTIONS - NATIVE_ACTIONS) | native_tools_for_capabilities(native) def extensions(self) -> list[ExtensionTool]: return [] # --- Coordinate helpers ---------------------------------------------------------- def _dims(self) -> tuple[int, int]: width = getattr(self.ctx.device, "device_width", 1080) if self.ctx.device else 1080 height = getattr(self.ctx.device, "device_height", 2400) if self.ctx.device else 2400 return width or 1080, height or 2400 def _to_px(self, nx: int, ny: int) -> tuple[int, int]: width, height = self._dims() x = int(max(0, min(width - 1, int(nx) * width / 1000))) y = int(max(0, min(height - 1, int(ny) * height / 1000))) return x, y # --- Required action ------------------------------------------------------------- async def _native_action(self, name: str, args: dict[str, Any]) -> ActionResult: from artemis.mcp.native_actions import READ_ONLY_ACTIONS, validate_native_action from artemis.mcp.native_executor import execute_native try: request = validate_native_action(name, args, *self._dims()) except ValueError as exc: return ActionResult.failure( name, str(exc), code=ActionCode.INVALID_ARGS, metadata={"execution": "not_started", "side_effect": "none"}, ) if name not in self.capabilities(): return ActionResult.failure( name, "The connected backend does not advertise this native action.", code=ActionCode.UNSUPPORTED, metadata={"execution": "not_started", "side_effect": "none"}, ) if resource_id := getattr(request, "require_visible", None): try: guard = await self._visibility_guard(resource_id) except (OSError, RuntimeError, ValueError) as exc: return ActionResult.failure( name, f"Visibility check failed before input: {exc}", metadata={"execution": "not_started", "side_effect": "none"}, ) if not guard["visible"]: return ActionResult.failure( name, "Required native control is not visible; no input executed.", code=ActionCode.TARGET_NOT_FOUND, metadata={ "execution": "not_started", "side_effect": "none", "visibility_guard": guard, }, ) receipt = await execute_native(name, request, *self._dims(), self.controller.native_request) receipt["read_only"] = name in READ_ONLY_ACTIONS receipt["side_effect"] = ( "none" if name in READ_ONLY_ACTIONS or receipt.get("execution") in {"not_started", "observed"} else "possible" ) if receipt.get("success"): return ActionResult.success( name, "Native operation completed. Inspect the receipt and current app state; business verification remains separate.", metadata=receipt, ) codes = { "UNSUPPORTED": ActionCode.UNSUPPORTED, "NOT_FOUND": ActionCode.TARGET_NOT_FOUND, "INVALID_ARGUMENT": ActionCode.INVALID_ARGS, "TIMEOUT": ActionCode.TIMEOUT, } return ActionResult.failure( name, str(receipt.get("error") or receipt.get("code") or "Native action failed."), code=codes.get(receipt.get("code"), ActionCode.DEVICE_ERROR), metadata=receipt, ) async def pinch(self, **kwargs: Any) -> ActionResult: return await self._native_action("pinch", kwargs) async def touch_gesture(self, **kwargs: Any) -> ActionResult: return await self._native_action("touch_gesture", kwargs) async def read_control(self, **kwargs: Any) -> ActionResult: return await self._native_action("read_control", kwargs) async def control_action(self, **kwargs: Any) -> ActionResult: return await self._native_action("control_action", kwargs) async def wait_until(self, **kwargs: Any) -> ActionResult: return await self._native_action("wait_until", kwargs) async def scroll_to(self, **kwargs: Any) -> ActionResult: return await self._native_action("scroll_to", kwargs) async def drag(self, **kwargs: Any) -> ActionResult: """Dispatch exact pixel geometry without a normalized-coordinate round trip.""" from artemis.mcp.gestures import DragRequest, plan_drag try: planned = plan_drag(DragRequest.model_validate(kwargs), *self._dims()) except ValueError as exc: return ActionResult.failure("drag", str(exc), code=ActionCode.INVALID_ARGS) return await self._dispatch_drag(planned) async def _dispatch_drag(self, planned: dict) -> ActionResult: metadata = dict(planned) metadata.pop("action") if resource_id := planned.get("require_visible"): try: guard = await self._visibility_guard(resource_id) except (OSError, RuntimeError) as exc: return ActionResult.failure( "drag", f"Visibility check failed; no input executed: {exc}", metadata={**metadata, "execution": "not_started", "side_effect": "none"}, ) metadata["visibility_guard"] = guard if not guard["visible"]: return ActionResult.failure( "drag", "TARGET_NOT_FOUND: required panel control has disappeared; no drag executed. Re-establish the panel and operate it in one guarded action_sequence.", code=ActionCode.TARGET_NOT_FOUND, metadata={**metadata, "execution": "not_started", "side_effect": "none"}, ) error = await self.controller.swipe_coords(*planned["coordinates"], planned["duration_ms"]) if error: return ActionResult.failure("drag", str(error), metadata=metadata) return ActionResult.success( "drag", f"Dragged from {planned['coordinates'][:2]} to {planned['coordinates'][2:]}" f" (pixel) over {planned['duration_ms']}ms. Verify the scale/value in the next observation.", duration_ms=planned["duration_ms"], metadata=metadata, ) async def _live_ui_elements(self) -> list[dict]: elements = await self.controller.get_ui_elements() if not isinstance(elements, list) or not elements: raise RuntimeError("No live UI hierarchy available for the visibility guard.") return elements async def _visibility_guard(self, resource_id: str, elements: list[dict] | None = None) -> dict: """Read live UI locally; never treat failed/empty hierarchy as absence.""" if elements is None: elements = await self._live_ui_elements() visible = any( (e.get("resource_id") or e.get("resource-id")) == resource_id and e.get("visible-to-user", True) not in (False, "false") for e in elements if isinstance(e, dict) ) return {"resource_id": resource_id, "visible": visible, "checked_at": time.time()} def _live_target_coordinates( self, resource_id: str, elements: list[dict], expected_point: tuple[int, int] | None = None ) -> list[int]: from artemis.utils.visualization import parse_bounds width, height = self._dims() matches = [] for element in elements: if ( element.get("resource_id") or element.get("resource-id") ) != resource_id or element.get("visible-to-user", True) in (False, "false"): continue raw_bounds = element.get("bounds") bounds = parse_bounds(raw_bounds) if isinstance(raw_bounds, str) else raw_bounds if not ( isinstance(bounds, (list, tuple)) and len(bounds) == 4 and all(isinstance(v, int) and not isinstance(v, bool) for v in bounds) ): continue if bounds: left, top, right, bottom = bounds left, top = max(0, left), max(0, top) right, bottom = min(width, right), min(height, bottom) if expected_point and not ( left <= expected_point[0] < right and top <= expected_point[1] < bottom ): continue if left < right and top < bottom: matches.append([int((left + right) / 2), int((top + bottom) / 2)]) if len(matches) != 1: raise LookupError( f"TARGET_NOT_FOUND: {resource_id} has {len(matches)} visible matches; no further tap executed. Use a unique observed resource_id or re-observed coordinates." ) return matches[0] async def _tap_native_target( self, resource_id: str, receipt: dict, *, times: int = 1, delay_ms: int = 100, elements: list[dict] | None = None, expected_point: tuple[int, int] | None = None, ) -> str | None: """Revalidate every repeated native tap, retaining partial input receipts.""" receipt.update(completed_taps=0, execution="not_started") for index in range(times): if index: await asyncio.sleep(max(0, delay_ms) / 1000) elements = None if elements is None: elements = await self._live_ui_elements() point = self._live_target_coordinates(resource_id, elements, expected_point) coordinate_key = "pixel_coordinates" if expected_point is not None else "coordinates" point = list(expected_point) if expected_point is not None else point receipt[coordinate_key] = point receipt["execution"] = "unknown" result = await self.controller.tap_at(*point) if error := getattr(result, "error", None): return str(error) receipt["completed_taps"] += 1 return None async def _ensure_visible_target(self, step: dict, receipt: dict) -> None: """Prepare explicit UI once; poll only reads, never retry the opener.""" receipt.update(execution="not_started", completed_taps=0, visibility_attempts=0) elements = await self._live_ui_elements() guard = await self._visibility_guard(step["target"], elements) receipt["visibility_guard"] = guard receipt["visibility_attempts"] += 1 if guard["visible"]: self._live_target_coordinates(step["target"], elements) receipt["execution"] = "skipped" return error = await self._tap_native_target(step["opener"], receipt, elements=elements) if error: raise RuntimeError(error) try: async with asyncio.timeout(step["timeout_ms"] / 1000): while True: elements = await self._live_ui_elements() guard = await self._visibility_guard(step["target"], elements) receipt["visibility_guard"] = guard receipt["visibility_attempts"] += 1 if guard["visible"]: self._live_target_coordinates(step["target"], elements) receipt["execution"] = "completed" return await asyncio.sleep(0.1) except TimeoutError as exc: raise TimeoutError( f"Panel target {step['target']} did not become visible within {step['timeout_ms']}ms after one opener tap. No opener retry; re-observe before continuing." ) from exc async def action_sequence(self, steps: list[Any]) -> ActionResult: """Execute once; optional visibility guards read live UI without model turns.""" import time from artemis.config.agent import load_agent_config from artemis.data_engine.trace import TraceSpan from artemis.mcp.gestures import plan_sequence, validate_sequence from artemis.runtime.task_control import TaskControl, has_pending_instruction try: limit = load_agent_config().pro.execution.max_burst_actions planned = plan_sequence( validate_sequence({"steps": steps}, limit), *self._dims(), limit=limit, ) except ValueError as exc: return ActionResult.failure("action_sequence", str(exc), code=ActionCode.INVALID_ARGS) from artemis.mcp.native_actions import NATIVE_ACTIONS unavailable = { step["action"] for step in planned if step["action"] in NATIVE_ACTIONS and step["action"] not in self.capabilities() } if unavailable: return ActionResult.failure( "action_sequence", f"Unsupported members {sorted(unavailable)}; no input executed.", code=ActionCode.UNSUPPORTED, metadata={"execution": "not_started", "side_effect": "none"}, ) started = time.monotonic() receipts = [ {**step, "index": i + 1, "execution": "not_started"} for i, step in enumerate(planned) ] failed = None failed_code = ActionCode.DEVICE_ERROR for step, receipt in zip(planned, receipts, strict=True): control = getattr(self.ctx, "task_control", None) if isinstance(control, TaskControl) and ( has_pending_instruction(control.directory) or (control.directory / "pause.request").exists() ): failed = "Sequence interrupted before the next input; re-observe before continuing." break with TraceSpan( name=f"action_sequence.{step['action']}", trace_type="tool", ctx=self.ctx ) as span: span.payload = {"args": step} receipt["started_ms"] = round((time.monotonic() - started) * 1000, 3) receipt["execution"] = "unknown" member_execution = "completed" try: if step["action"] == "ensure_visible": try: await self._ensure_visible_target(step, receipt) member_execution = receipt["execution"] error = None except (LookupError, TimeoutError) as exc: error = str(exc) member_execution = receipt["execution"] failed_code = ( ActionCode.TIMEOUT if isinstance(exc, TimeoutError) else ActionCode.TARGET_NOT_FOUND ) elif step["action"] == "click": guard = None elements = None error = None if step.get("unless_visible") or step.get("target_resource_id"): receipt["execution"] = "not_started" elements = await self._live_ui_elements() if resource_id := step.get("unless_visible"): guard = await self._visibility_guard(resource_id, elements) receipt["visibility_guard"] = guard if guard and guard["visible"]: member_execution = "skipped" error = None else: if resource_id := step.get("target_resource_id"): try: error = await self._tap_native_target( resource_id, receipt, times=step.get("times", 1), delay_ms=step.get("delay_ms", 100), elements=elements, ) except LookupError as exc: error = str(exc) member_execution = receipt["execution"] failed_code = ActionCode.TARGET_NOT_FOUND else: receipt["execution"] = "unknown" repeats = ( {"times": step["times"], "delay_ms": step["delay_ms"]} if "times" in step else {} ) result = await self.controller.tap_at( *receipt["coordinates"], **repeats ) error = getattr(result, "error", None) elif step["action"] in NATIVE_ACTIONS: result = await self._native_action(step["action"], step["args"]) error = None if result.ok else result.message receipt.update(result.metadata) receipt["native_execution"] = result.metadata.get("execution", "unknown") if not result.ok: member_execution = result.metadata.get("execution", "unknown") failed_code = result.code elif step["action"] == "drag": result = await self._dispatch_drag(step) error = None if result.ok else result.message if not result.ok: member_execution = result.metadata.get("execution", "unknown") failed_code = result.code if guard_result := result.metadata.get("visibility_guard"): receipt["visibility_guard"] = guard_result else: await asyncio.sleep(step["time_in_ms"] / 1000) error = None receipt.update( execution=( member_execution if member_execution in {"skipped", "not_started", "cancelled", "observed"} or not error else "unknown" ), error=error, ) if error: failed = str(error) span.status = "failed" span.error = failed except asyncio.CancelledError: receipt.update(execution="unknown", error="Input cancelled; effect uncertain.") raise except (OSError, RuntimeError) as exc: receipt.update(error=str(exc)) failed = str(exc) span.status = "failed" span.error = failed finally: receipt["ended_ms"] = round((time.monotonic() - started) * 1000, 3) span.result = dict(receipt) span.payload["receipt"] = dict(receipt) if failed: break completed = sum(r["execution"] == "completed" for r in receipts) skipped = sum(r["execution"] == "skipped" for r in receipts) attempted = any( r["action"] not in {"wait", "wait_until"} and r["execution"] in {"completed", "unknown", "cancelled"} for r in receipts ) execution = "completed" if not failed else ("partial" if completed else "unknown") if failed and not attempted: execution = "not_started" metadata = { "sequence_results": receipts, "completed_steps": completed, "skipped_steps": skipped, "execution": execution, "side_effect": "possible" if attempted else "none", } if failed: return ActionResult.failure( "action_sequence", f"Sequence stopped after {completed}/{len(planned)} completed steps: {failed}" " Remaining steps were not executed; do not replay completed inputs.", code=failed_code, metadata=metadata, ) return ActionResult.success( "action_sequence", f"Executed {completed} gesture steps consecutively; skipped {skipped} unnecessary UI-opening steps. Verify the intended result in the final screen.", metadata=metadata, ) async def click_sequence( self, points: list[tuple[int, int]], delay_ms: int = 50 ) -> ActionResult: outcomes = [] for i, (nx, ny) in enumerate(points): x, y = self._to_px(nx, ny) result = await self.controller.tap_at(x, y) err = getattr(result, "error", None) if result else None if err: return ActionResult.failure( "click_sequence", f"Error executing click at step {i + 1}: {err}", detail=str(err), ) outcomes.append(f"[{nx}, {ny}]") if i < len(points) - 1: await asyncio.sleep(max(0, delay_ms) / 1000.0) return ActionResult.success( "click_sequence", f"Tapped in sequence at {'; '.join(outcomes)} (normalized).", ) # --- Optional actions ------------------------------------------------------------ async def click( self, nx: int, ny: int, times: int = 1, delay_ms: int = 100, *, expected_resource_id: str | None = None, ) -> ActionResult: x, y = self._to_px(nx, ny) if expected_resource_id: receipt: dict = {} try: error = await self._tap_native_target( expected_resource_id, receipt, times=times, delay_ms=delay_ms, expected_point=(x, y), ) except (LookupError, OSError, RuntimeError) as exc: return ActionResult.failure( "click", f"Observed target could not be confirmed for the next tap; completed {receipt.get('completed_taps', 0)} taps: {exc}. Re-observe. For an auto-hiding panel use one guarded action_sequence containing its opener and target, using observed resource IDs.", code=ActionCode.TARGET_NOT_FOUND if isinstance(exc, LookupError) else ActionCode.DEVICE_ERROR, metadata={ **receipt, "side_effect": "none" if receipt.get("execution") == "not_started" else "possible", "expected_resource_id": expected_resource_id, }, ) if error: return ActionResult.failure("click", error, metadata=receipt) return ActionResult.success( "click", f"Tapped at [{nx}, {ny}] (normalized).", normalized_coordinates=[int(nx), int(ny)], metadata={**receipt, "execution": "completed"}, ) result = await self.controller.tap_at(x, y, times=times, delay_ms=delay_ms) err = getattr(result, "error", None) if result else None if err is not None: return ActionResult.failure("click", f"Error executing click: {err}", detail=str(err)) return ActionResult.success( "click", f"Tapped at [{nx}, {ny}] (normalized).", normalized_coordinates=[int(nx), int(ny)], ) async def long_press(self, nx: int, ny: int, duration_ms: int = 1000) -> ActionResult: x, y = self._to_px(nx, ny) result = await self.controller.tap_at( x, y, long_press=True, long_press_duration=duration_ms ) err = getattr(result, "error", None) if result else None if err is not None: return ActionResult.failure( "long_press", f"Error executing long press: {err}", detail=str(err) ) return ActionResult.success( "long_press", f"Long-pressed at [{nx}, {ny}] (normalized) for {duration_ms}ms.", normalized_coordinates=[int(nx), int(ny)], duration_ms=duration_ms, ) async def input_text( self, text: str, target: tuple[int, int] | None = None, clear_exist: bool = True, ) -> ActionResult: coords = None if target: nx, ny = target x, y = self._to_px(nx, ny) coords = [int(nx), int(ny)] err = await ensure_focus_at_coords(self.controller, x, y) if err: return ActionResult.failure( "input_text", f"Error focusing element: {err}", detail=str(err), normalized_coordinates=coords, ) if clear_exist: if hasattr(self.controller, "erase_text") and not await self.controller.erase_text(): return ActionResult.failure( "input_text", "Failed to clear existing text", normalized_coordinates=coords, ) elif hasattr(self.controller, "press_key"): # KEYCODE_MOVE_END: move cursor to the end for reliable append. await self.controller.press_key("123") if hasattr(self.controller, "type_text") and not await self.controller.type_text( text, clear_existing=False ): return ActionResult.failure( "input_text", f"Failed typing '{text}'.", normalized_coordinates=coords, ) return ActionResult.success( "input_text", f"Typed '{text}'" + (f" at [{coords[0]}, {coords[1]}] (normalized)." if coords else "."), normalized_coordinates=coords, ) async def swipe( self, start: tuple[int, int], end: tuple[int, int], duration_ms: int = 800, ) -> ActionResult: nx1, ny1 = start nx2, ny2 = end px1, py1 = self._to_px(nx1, ny1) px2, py2 = self._to_px(nx2, ny2) err = await self.controller.swipe_coords(px1, py1, px2, py2, duration_ms) if err: return ActionResult.failure("swipe", f"Error dragging: {err}", detail=str(err)) return ActionResult.success( "swipe", f"Swiped from [{nx1}, {ny1}] to [{nx2}, {ny2}] (normalized).", normalized_coordinates=[int(nx1), int(ny1), int(nx2), int(ny2)], duration_ms=duration_ms, ) async def press_key(self, key: str) -> ActionResult: key_str = str(key).lower() if not key_str: return ActionResult.failure( "press_key", f"Error executing key press '{key}'.", code=ActionCode.INVALID_ARGS, ) if key_str == "back" and hasattr(self.controller, "go_back"): await self.controller.go_back() elif key_str == "home" and hasattr(self.controller, "go_home"): await self.controller.go_home() elif key_str == "enter" and hasattr(self.controller, "press_enter"): await self.controller.press_enter() elif hasattr(self.controller, "press_key"): # Pass unrecognized keycodes through verbatim (KEYCODE_* names, numeric # codes): the driver resolves or forwards them, matching the historical # adb_server behavior of accepting any Android key event. res = await self.controller.press_key(key) err = getattr(res, "error", None) if res else None if err: return ActionResult.failure( "press_key", f"Error executing key press '{key}': {err}", detail=str(err), ) if not res: return ActionResult.failure("press_key", f"Error executing key press '{key}'.") return ActionResult.success("press_key", f"Pressed key '{key}'.") async def manage_app(self, action: str, app_name: str) -> ActionResult: # Imported lazily: launch_app pulls in tool wrappers that are costly at import. from artemis.tools.mobile.launch_app import find_package from artemis.utils.app_launch_utils import launch_app_with_retries res = find_package(self.ctx, app_name, use_fallback=False) pkg = await res if inspect.iscoroutine(res) else res if not pkg: return ActionResult.failure( "manage_app", f"Error finding package for app: {app_name}", code=ActionCode.PACKAGE_NOT_FOUND, ) target_pkg = pkg if action.lower() == "launch": res_launch = launch_app_with_retries(self.ctx, target_pkg) if inspect.iscoroutine(res_launch): res_launch = await res_launch if isinstance(res_launch, tuple): success, error_msg = res_launch else: success, error_msg = bool(res_launch), "" if success: return ActionResult.success( "manage_app", f"Launched app '{app_name}' ({target_pkg}); foreground confirmed.", ) return ActionResult.failure( "manage_app", f"Failed to launch app '{app_name}': {error_msg}", detail=error_msg or None, ) elif action.lower() == "stop": if hasattr(self.controller, "terminate_app"): res_term = self.controller.terminate_app(target_pkg) if inspect.iscoroutine(res_term): await res_term return ActionResult.success( "manage_app", f"Force-stopped app '{app_name}' ({target_pkg})." ) return ActionResult.failure( "manage_app", f"Invalid manage_app action: {action}", code=ActionCode.INVALID_ARGS, ) async def wait_for_delay(self, time_in_ms: int) -> ActionResult: delay_s = max(0.0, float(time_in_ms) / 1000.0) await asyncio.sleep(delay_s) return ActionResult.success( "wait_for_delay", f"Waited {time_in_ms}ms.", duration_ms=int(time_in_ms), ) async def wait_for_text( self, text: str, wait_state: str | None = None, timeout_ms: int | None = None ) -> ActionResult: timeout_s = (timeout_ms or 5000) / 1000.0 start = time.time() found = False target_state = (wait_state or "appear").lower() while time.time() - start < timeout_s: tree = await self.controller.get_ui_elements() text_present = text.lower() in str(tree or "").lower() if target_state == "appear" and text_present: found = True break elif target_state == "disappear" and not text_present: found = True break await asyncio.sleep(0.5) if found: return ActionResult.success( "wait_for_text", f"Text '{text}' {'appeared' if target_state == 'appear' else 'disappeared'}" f" in the UI tree after {int((time.time() - start) * 1000)}ms.", ) return ActionResult.failure( "wait_for_text", f"Timed out waiting for text '{text}' to {target_state}.", code=ActionCode.TIMEOUT, ) async def open_link(self, url: str) -> ActionResult: success = await self.controller.open_url(url) if success: return ActionResult.success("open_link", f"Opened link '{url}'.") return ActionResult.failure("open_link", f"Failed to open link '{url}'.") async def erase_one_char(self) -> ActionResult: success = await self.controller.erase_text(nb_chars=1) if success: return ActionResult.success("erase_one_char", "Erased one character.") return ActionResult.failure("erase_one_char", "Failed to erase one character.") async def focus_and_clear_text(self, nx: int, ny: int) -> ActionResult: x, y = self._to_px(nx, ny) err = await ensure_focus_at_coords(self.controller, x, y) if err: return ActionResult.failure( "focus_and_clear_text", f"Error focusing element: {err}", detail=str(err), normalized_coordinates=[int(nx), int(ny)], ) success = await self.controller.erase_text() if success: return ActionResult.success( "focus_and_clear_text", f"Cleared text at [{nx}, {ny}] (normalized).", normalized_coordinates=[int(nx), int(ny)], ) return ActionResult.failure("focus_and_clear_text", "Failed to erase text.") # --- Internal observation primitives --------------------------------------------- async def take_screenshot(self) -> str: return await self.controller.take_screenshot() async def get_ui_elements(self) -> Any: return await self.controller.get_ui_elements() async def get_screen_data(self) -> Any: return await self.controller.get_screen_data()