feat: initial release of ARTEMIS Android Agent & Automation Platform
ARTEMIS is an autonomous multimodal Android agent and mobile UI automation platform engineered for physical devices and emulators. It combines state-of-the-art multimodal Vision-Language Models (VLMs) with hierarchical closed-loop planning, resilient self-healing, and deep Android subsystem integration. Key Features & Architectural Highlights: - Dual-Engine Orchestration: * Pro Engine: Multi-agent closed-loop (Planner -> Operator -> Validator/Failure Analyzer -> Summarizer) powered by LangGraph. * Flash Engine: Ultra-fast reactive loop for deterministic, low-latency UI execution (<35 steps). - Multimodal Perception & Grounding: * Hybrid visual screenshot grounding, real-time OCR extraction, and XML accessibility hierarchy parsing. * Dynamic-First, Coordinate-Fallback locating strategy for maximum resilience against UI drift. - Autonomous Self-Healing & Diagnostics: * Failure analyzer and safety net for popup dismissal, crash recovery, and auto-retry. * Diagnostic sub-agents for log analysis, video recording inspection, and object detection. - Unified CLI & Python SDK: * Rich CLI with commands: init, doctor, run, trace, batch, bench, server, and mcp. * Fluent Python SDK with Pydantic structured output models and package locking. - Multi-Provider LLM Architecture: * Support for Google Gemini, OpenAI, Anthropic Claude, Vertex AI, OpenRouter, xAI Grok, and local endpoints (Ollama/vLLM). * Centralized configuration in config/artemis.jsonc. - Native MCP & Observability: * Model Context Protocol (MCP) server endpoints for AI IDEs. * Full trace recording with before/after screenshots, action overlays, and execution replays. Change-Id: I430b93ee2b76b54cc5a3c6eeccd15d2cf29830a1
This commit is contained in:
somew1nd committed
commit
206a9ce921
229 files changed
+49653
No files matched your search
@@ -0,0 +1,481 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Concrete Android Device Driver implementation using ADB and UIAutomator2."""
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
|
||||
from adbutils import AdbClient, AdbDevice
|
||||
from artemis.clients.ui_automator_client import UIAutomatorClient
|
||||
from artemis.drivers.base import BaseDeviceDriver, KeyCode, ScreenData, SwipeDirection
|
||||
from artemis.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
# KeyCode mapping from string name to Android keycode integer
|
||||
ANDROID_KEYCODE_MAP: dict[str, int] = {
|
||||
"home": 3,
|
||||
"back": 4,
|
||||
"enter": 66,
|
||||
"delete": 67,
|
||||
"power": 26,
|
||||
"app_switch": 187,
|
||||
"volume_up": 24,
|
||||
"volume_down": 25,
|
||||
}
|
||||
|
||||
|
||||
def _escape_for_adb_text(s: str) -> str:
|
||||
"""Escapes special characters for adb shell input text."""
|
||||
return (
|
||||
s.replace("\\", "\\\\")
|
||||
.replace('"', '\\"')
|
||||
.replace("'", "\\'")
|
||||
.replace("`", "\\`")
|
||||
.replace("$", "\\$")
|
||||
.replace("&", "\\&")
|
||||
.replace("|", "\\|")
|
||||
.replace(";", "\\;")
|
||||
.replace("<", "\\<")
|
||||
.replace(">", "\\>")
|
||||
.replace("(", "\\(")
|
||||
.replace(")", "\\)")
|
||||
.replace("*", "\\*")
|
||||
.replace("?", "\\?")
|
||||
.replace("~", "\\~")
|
||||
.replace(" ", "%s")
|
||||
)
|
||||
|
||||
|
||||
class AndroidAdbDriver(BaseDeviceDriver):
|
||||
"""Production-grade Android Device Driver powered by ADB and UIAutomator2."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
device_id: str,
|
||||
adb_client: AdbClient,
|
||||
ui_adb_client: UIAutomatorClient | None = None,
|
||||
width: int = 1080,
|
||||
height: int = 2400,
|
||||
):
|
||||
self._device_id = device_id
|
||||
self._adb_client = adb_client
|
||||
self._ui_adb_client = ui_adb_client
|
||||
self._width = width
|
||||
self._height = height
|
||||
self._device: AdbDevice | None = None
|
||||
self._recording_process: asyncio.subprocess.Process | None = None
|
||||
self._recording_output_path: Path | None = None
|
||||
|
||||
@property
|
||||
def device_id(self) -> str:
|
||||
return self._device_id
|
||||
|
||||
@property
|
||||
def screen_size(self) -> tuple[int, int]:
|
||||
return (self._width, self._height)
|
||||
|
||||
@property
|
||||
def device(self) -> AdbDevice:
|
||||
if self._device is None:
|
||||
self._device = self._adb_client.device(serial=self._device_id)
|
||||
return self._device
|
||||
|
||||
async def connect(self) -> None:
|
||||
"""Verify device availability."""
|
||||
try:
|
||||
state = self.device.get_state()
|
||||
logger.info(f"Android device '{self._device_id}' connected (state: {state})")
|
||||
except Exception as e:
|
||||
logger.warning(f"Connecting to Android device '{self._device_id}': {e}")
|
||||
|
||||
async def disconnect(self) -> None:
|
||||
"""Clean up ongoing recordings and tunnels."""
|
||||
if self._recording_process:
|
||||
await self.stop_video_recording()
|
||||
|
||||
async def get_screen_data(self, skip_settling: bool = False) -> ScreenData:
|
||||
"""Captures screenshot and XML hierarchy concurrently."""
|
||||
if not skip_settling:
|
||||
await asyncio.sleep(0.3)
|
||||
|
||||
# 1. Capture raw screenshot
|
||||
screenshot_bytes: bytes = b""
|
||||
screenshot_base64: str = ""
|
||||
|
||||
# Check if ui_adb_client has direct screen data support
|
||||
if self._ui_adb_client and hasattr(self._ui_adb_client, "get_screen_data"):
|
||||
try:
|
||||
raw_ui_data = self._ui_adb_client.get_screen_data()
|
||||
if (
|
||||
hasattr(raw_ui_data, "base64")
|
||||
and isinstance(raw_ui_data.base64, str)
|
||||
and raw_ui_data.base64
|
||||
):
|
||||
screenshot_base64 = raw_ui_data.base64
|
||||
screenshot_bytes = base64.b64decode(screenshot_base64)
|
||||
if hasattr(raw_ui_data, "width") and isinstance(raw_ui_data.width, int):
|
||||
self._width = raw_ui_data.width
|
||||
if hasattr(raw_ui_data, "height") and isinstance(raw_ui_data.height, int):
|
||||
self._height = raw_ui_data.height
|
||||
except Exception as e:
|
||||
logger.debug(f"Direct ui_adb_client.get_screen_data error: {e}")
|
||||
|
||||
if not screenshot_base64 or not isinstance(screenshot_base64, str):
|
||||
try:
|
||||
pil_img = await asyncio.to_thread(self.device.screenshot)
|
||||
if hasattr(pil_img, "save"):
|
||||
buf = BytesIO()
|
||||
pil_img.save(buf, format="PNG")
|
||||
screenshot_bytes = buf.getvalue()
|
||||
self._width, self._height = pil_img.size
|
||||
screenshot_base64 = base64.b64encode(screenshot_bytes).decode("utf-8")
|
||||
except Exception as e:
|
||||
logger.warning(f"Device screenshot capture failed on {self._device_id}: {e}")
|
||||
|
||||
if not screenshot_base64 or not isinstance(screenshot_base64, str):
|
||||
# Fallback for headless testing environments
|
||||
screenshot_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="
|
||||
screenshot_bytes = base64.b64decode(screenshot_base64)
|
||||
|
||||
# 2. Capture UI Hierarchy via UIAutomator
|
||||
ui_hierarchy_xml: str | None = None
|
||||
ui_elements: list[dict[str, Any]] = []
|
||||
if self._ui_adb_client:
|
||||
try:
|
||||
if hasattr(self._ui_adb_client, "get_hierarchy"):
|
||||
res = self._ui_adb_client.get_hierarchy()
|
||||
ui_elements = await res if asyncio.iscoroutine(res) else res
|
||||
elif hasattr(self._ui_adb_client, "get_ui_elements"):
|
||||
res = self._ui_adb_client.get_ui_elements()
|
||||
ui_elements = await res if asyncio.iscoroutine(res) else res
|
||||
except Exception as e:
|
||||
logger.debug(f"UI hierarchy extraction failed: {e}")
|
||||
|
||||
if isinstance(ui_elements, dict):
|
||||
clean_ui_elements = [ui_elements]
|
||||
elif isinstance(ui_elements, list):
|
||||
clean_ui_elements = ui_elements
|
||||
else:
|
||||
clean_ui_elements = []
|
||||
|
||||
w = self._width if isinstance(self._width, int) else 1080
|
||||
h = self._height if isinstance(self._height, int) else 2400
|
||||
clean_b64 = (
|
||||
screenshot_base64
|
||||
if isinstance(screenshot_base64, str) and screenshot_base64
|
||||
else "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="
|
||||
)
|
||||
clean_bytes = (
|
||||
screenshot_bytes
|
||||
if isinstance(screenshot_bytes, bytes) and screenshot_bytes
|
||||
else base64.b64decode(clean_b64)
|
||||
)
|
||||
|
||||
return ScreenData(
|
||||
screenshot_bytes=clean_bytes,
|
||||
screenshot_base64=clean_b64,
|
||||
ui_hierarchy_xml=ui_hierarchy_xml,
|
||||
ui_elements=clean_ui_elements,
|
||||
width=w,
|
||||
height=h,
|
||||
platform="android",
|
||||
)
|
||||
|
||||
async def tap(
|
||||
self,
|
||||
x: int,
|
||||
y: int,
|
||||
duration_ms: int = 100,
|
||||
times: int = 1,
|
||||
delay_ms: int = 100,
|
||||
) -> bool:
|
||||
try:
|
||||
if duration_ms >= 500:
|
||||
cmd = f"input swipe {x} {y} {x} {y} {duration_ms}"
|
||||
else:
|
||||
if times <= 1:
|
||||
cmd = f"input tap {x} {y}"
|
||||
else:
|
||||
taps = [f"input tap {x} {y}"] * times
|
||||
cmd = f" && sleep {delay_ms / 1000.0:.3f} && ".join(taps)
|
||||
|
||||
logger.info(f"[ADB] {cmd}")
|
||||
await asyncio.to_thread(self.device.shell, cmd)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Tap failed at ({x}, {y}): {e}")
|
||||
return False
|
||||
|
||||
async def long_press(self, x: int, y: int, duration_ms: int = 1000) -> bool:
|
||||
return await self.tap(x=x, y=y, duration_ms=duration_ms)
|
||||
|
||||
async def swipe(
|
||||
self,
|
||||
start_x: int,
|
||||
start_y: int,
|
||||
end_x: int,
|
||||
end_y: int,
|
||||
duration_ms: int = 400,
|
||||
) -> bool:
|
||||
try:
|
||||
cmd = f"input swipe {start_x} {start_y} {end_x} {end_y} {duration_ms}"
|
||||
logger.info(f"[ADB] {cmd}")
|
||||
await asyncio.to_thread(self.device.shell, cmd)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Swipe failed from ({start_x},{start_y}) to ({end_x},{end_y}): {e}")
|
||||
return False
|
||||
|
||||
async def swipe_direction(
|
||||
self,
|
||||
direction: SwipeDirection | Literal["up", "down", "left", "right"],
|
||||
duration_ms: int = 400,
|
||||
) -> bool:
|
||||
dir_str = str(direction).lower()
|
||||
mid_x = self._width // 2
|
||||
mid_y = self._height // 2
|
||||
|
||||
if dir_str == "up":
|
||||
# Drag bottom to top -> scrolls down
|
||||
return await self.swipe(
|
||||
mid_x, int(self._height * 0.75), mid_x, int(self._height * 0.25), duration_ms
|
||||
)
|
||||
elif dir_str == "down":
|
||||
# Drag top to bottom -> scrolls up
|
||||
return await self.swipe(
|
||||
mid_x, int(self._height * 0.25), mid_x, int(self._height * 0.75), duration_ms
|
||||
)
|
||||
elif dir_str == "left":
|
||||
# Drag right to left -> scrolls right
|
||||
return await self.swipe(
|
||||
int(self._width * 0.8), mid_y, int(self._width * 0.2), mid_y, duration_ms
|
||||
)
|
||||
elif dir_str == "right":
|
||||
# Drag left to right -> scrolls left
|
||||
return await self.swipe(
|
||||
int(self._width * 0.2), mid_y, int(self._width * 0.8), mid_y, duration_ms
|
||||
)
|
||||
return False
|
||||
|
||||
async def input_text(self, text: str, clear_existing: bool = True) -> bool:
|
||||
try:
|
||||
if clear_existing:
|
||||
# Safe & robust clearing: Move to End -> Shift+Home selection -> Delete -> Fallback backspaces
|
||||
clear_cmd = (
|
||||
"input keyevent 123 && "
|
||||
"input keyevent --meta 1 122 && "
|
||||
"input keyevent 67 && "
|
||||
"input keyevent 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67"
|
||||
)
|
||||
await asyncio.to_thread(self.device.shell, clear_cmd)
|
||||
else:
|
||||
# Append mode: move cursor to the very end of existing text
|
||||
await asyncio.to_thread(
|
||||
self.device.shell,
|
||||
"input keyevent 123", # KEYCODE_MOVE_END
|
||||
)
|
||||
|
||||
# 1. Fast path for ASCII text: Use ADB input text directly.
|
||||
# This avoids toggling IME, preserves keyboard focus, and works on all Android devices.
|
||||
if text.isascii():
|
||||
lines = text.split("\n")
|
||||
for i, line in enumerate(lines):
|
||||
if i > 0:
|
||||
# Send Enter key between lines
|
||||
await asyncio.to_thread(self.device.shell, "input keyevent 66")
|
||||
if line:
|
||||
escaped = _escape_for_adb_text(line)
|
||||
await asyncio.to_thread(self.device.shell, f"input text {escaped}")
|
||||
return True
|
||||
|
||||
# 2. For non-ASCII / Unicode text: Try UIAutomator2 FastInputIME
|
||||
if self._ui_adb_client and hasattr(self._ui_adb_client, "send_text"):
|
||||
try:
|
||||
res = self._ui_adb_client.send_text(text)
|
||||
if asyncio.iscoroutine(res):
|
||||
await res
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning(f"UIAutomator2 send_text failed: {e}")
|
||||
|
||||
# 3. Check if ADBKeyboard is currently active
|
||||
try:
|
||||
default_ime = await asyncio.to_thread(
|
||||
self.device.shell, "settings get secure default_input_method"
|
||||
)
|
||||
if "adbkeyboard" in str(default_ime).lower():
|
||||
b64_text = base64.b64encode(text.encode("utf-8")).decode("utf-8")
|
||||
broadcast_cmd = f"am broadcast -a ADB_INPUT_B64 --es msg '{b64_text}'"
|
||||
await asyncio.to_thread(self.device.shell, broadcast_cmd)
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 4. Fallback: ADB input text
|
||||
lines = text.split("\n")
|
||||
for i, line in enumerate(lines):
|
||||
if i > 0:
|
||||
await asyncio.to_thread(self.device.shell, "input keyevent 66")
|
||||
if line:
|
||||
escaped = _escape_for_adb_text(line)
|
||||
await asyncio.to_thread(self.device.shell, f"input text {escaped}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Input text failed for '{text}': {e}")
|
||||
return False
|
||||
|
||||
async def press_key(self, key: KeyCode | str | int) -> bool:
|
||||
try:
|
||||
keycode_val = key
|
||||
if isinstance(key, (KeyCode, str)):
|
||||
key_name = str(key).lower().replace("keycode.", "")
|
||||
keycode_val = ANDROID_KEYCODE_MAP.get(key_name, key)
|
||||
|
||||
await asyncio.to_thread(self.device.shell, f"input keyevent {keycode_val}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Press key failed for '{key}': {e}")
|
||||
return False
|
||||
|
||||
async def launch_app(self, package_name: str) -> bool:
|
||||
try:
|
||||
cmd = f"monkey -p {package_name} -c android.intent.category.LAUNCHER 1"
|
||||
await asyncio.to_thread(self.device.shell, cmd)
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Launch app failed for '{package_name}': {e}")
|
||||
return False
|
||||
|
||||
async def stop_app(self, package_name: str) -> bool:
|
||||
try:
|
||||
await asyncio.to_thread(self.device.shell, f"am force-stop {package_name}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Stop app failed for '{package_name}': {e}")
|
||||
return False
|
||||
|
||||
async def get_current_package(self) -> str | None:
|
||||
try:
|
||||
# 1. Try modern current_app from adbutils
|
||||
if hasattr(self.device, "current_app"):
|
||||
try:
|
||||
app_info = await asyncio.to_thread(self.device.current_app)
|
||||
if app_info and getattr(app_info, "package", None):
|
||||
return app_info.package
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 2. Structured dumpsys extraction
|
||||
out = await asyncio.to_thread(
|
||||
self.device.shell, "dumpsys window displays | grep -E 'mCurrentFocus|mFocusedApp'"
|
||||
)
|
||||
for line in str(out).splitlines():
|
||||
if "/" in line:
|
||||
for token in line.split():
|
||||
if "/" in token and "." in token:
|
||||
clean = token.split("/")[0].strip("{} ,")
|
||||
if clean and not clean.startswith("Window") and "." in clean:
|
||||
return clean
|
||||
except Exception as e:
|
||||
logger.debug(f"Error querying current package: {e}")
|
||||
return None
|
||||
|
||||
async def execute_shell(self, command: str, timeout_seconds: float = 15.0) -> str:
|
||||
try:
|
||||
return await asyncio.wait_for(
|
||||
asyncio.to_thread(self.device.shell, command),
|
||||
timeout=timeout_seconds,
|
||||
)
|
||||
except Exception as e:
|
||||
return f"Error: {e}"
|
||||
|
||||
async def start_video_recording(self, output_dir: Path | None = None) -> None:
|
||||
"""Starts screen recording via scrcpy in background."""
|
||||
out_dir = output_dir or Path("/tmp")
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
self._recording_mkv_path = out_dir / "recording.mkv"
|
||||
self._recording_output_path = out_dir / "recording.mp4"
|
||||
logger.info(f"Starting scrcpy video recording to {self._recording_mkv_path}...")
|
||||
try:
|
||||
cmd = [
|
||||
"scrcpy",
|
||||
"--serial",
|
||||
self.device_id,
|
||||
"--no-window",
|
||||
"--record",
|
||||
str(self._recording_mkv_path),
|
||||
"--record-format",
|
||||
"mkv",
|
||||
"--video-bit-rate",
|
||||
"2M",
|
||||
]
|
||||
self._scrcpy_process = await asyncio.create_subprocess_exec(
|
||||
*cmd,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
await asyncio.sleep(0.5)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to start scrcpy subprocess: {e}")
|
||||
|
||||
async def stop_video_recording(self) -> str | None:
|
||||
"""Stops background video capture and returns local recording file path."""
|
||||
if hasattr(self, "_scrcpy_process") and self._scrcpy_process:
|
||||
try:
|
||||
self._scrcpy_process.terminate()
|
||||
await asyncio.wait_for(self._scrcpy_process.wait(), timeout=5.0)
|
||||
except Exception:
|
||||
pass
|
||||
self._scrcpy_process = None
|
||||
|
||||
mkv = getattr(self, "_recording_mkv_path", None)
|
||||
mp4 = getattr(self, "_recording_output_path", None)
|
||||
|
||||
if mkv and mkv.exists() and mp4:
|
||||
try:
|
||||
from artemis.utils.video import get_ffmpeg_path
|
||||
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
get_ffmpeg_path(),
|
||||
"-y",
|
||||
"-i",
|
||||
str(mkv),
|
||||
"-c",
|
||||
"copy",
|
||||
"-movflags",
|
||||
"+faststart",
|
||||
str(mp4),
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
await proc.wait()
|
||||
if mp4.exists() and mp4.stat().st_size > 0:
|
||||
try:
|
||||
mkv.unlink()
|
||||
except Exception:
|
||||
pass
|
||||
return str(mp4)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to convert MKV to MP4 in AdbDriver: {e}")
|
||||
|
||||
if mp4 and mp4.exists():
|
||||
return str(mp4)
|
||||
if mkv and mkv.exists():
|
||||
return str(mkv)
|
||||
return None
|
||||
@@ -0,0 +1,64 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Android XML Layout Hierarchy parser and semantic node extractor."""
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
import xml.etree.ElementTree as ET
|
||||
from artemis.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
def parse_ui_hierarchy(xml_string: str) -> list[dict[str, Any]]:
|
||||
"""Parses raw Android XML hierarchy dump into structured interactive UI nodes."""
|
||||
if not xml_string:
|
||||
return []
|
||||
|
||||
elements: list[dict[str, Any]] = []
|
||||
try:
|
||||
root = ET.fromstring(xml_string)
|
||||
for node in root.iter():
|
||||
attribs = node.attrib
|
||||
bounds_str = attribs.get("bounds", "")
|
||||
match = re.match(r"\[(\d+),(\d+)\]\[(\d+),(\d+)\]", bounds_str)
|
||||
if not match:
|
||||
continue
|
||||
|
||||
x1, y1, x2, y2 = map(int, match.groups())
|
||||
# Filter zero-area nodes
|
||||
if x1 >= x2 or y1 >= y2:
|
||||
continue
|
||||
|
||||
text = attribs.get("text") or attribs.get("content-desc") or ""
|
||||
class_name = attribs.get("class", "")
|
||||
resource_id = attribs.get("resource-id", "")
|
||||
clickable = attribs.get("clickable") == "true"
|
||||
|
||||
if text or clickable or resource_id:
|
||||
elements.append(
|
||||
{
|
||||
"text": text,
|
||||
"resource_id": resource_id,
|
||||
"class": class_name,
|
||||
"clickable": clickable,
|
||||
"bounds": [x1, y1, x2, y2],
|
||||
"center": [(x1 + x2) // 2, (y1 + y2) // 2],
|
||||
}
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to parse XML hierarchy: {e}")
|
||||
|
||||
return elements
|
||||
@@ -0,0 +1,119 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Android IME and keyboard text typing helper."""
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
from typing import Any
|
||||
from artemis.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
def _escape_for_adb_text(s: str) -> str:
|
||||
"""Escapes special characters for adb shell input text."""
|
||||
return (
|
||||
s.replace("\\", "\\\\")
|
||||
.replace('"', '\\"')
|
||||
.replace("'", "\\'")
|
||||
.replace("`", "\\`")
|
||||
.replace("$", "\\$")
|
||||
.replace("&", "\\&")
|
||||
.replace("|", "\\|")
|
||||
.replace(";", "\\;")
|
||||
.replace("<", "\\<")
|
||||
.replace(">", "\\>")
|
||||
.replace("(", "\\(")
|
||||
.replace(")", "\\)")
|
||||
.replace("*", "\\*")
|
||||
.replace("?", "\\?")
|
||||
.replace("~", "\\~")
|
||||
.replace(" ", "%s")
|
||||
)
|
||||
|
||||
|
||||
class AndroidInputIME:
|
||||
"""Handles text input, unicode typing, and key combination injection."""
|
||||
|
||||
def __init__(self, adb_device: Any, ui_client: Any = None):
|
||||
self.device = adb_device
|
||||
self.ui_client = ui_client
|
||||
|
||||
async def type_text(self, text: str, clear_existing: bool = True) -> bool:
|
||||
"""Types text using adb input text, UIAutomator, or broadcast IME."""
|
||||
try:
|
||||
if clear_existing:
|
||||
# Safe & robust clearing: Move to End -> Shift+Home selection -> Delete -> Fallback backspaces
|
||||
clear_cmd = (
|
||||
"input keyevent 123 && "
|
||||
"input keyevent --meta 1 122 && "
|
||||
"input keyevent 67 && "
|
||||
"input keyevent 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67"
|
||||
)
|
||||
await asyncio.to_thread(self.device.shell, clear_cmd)
|
||||
else:
|
||||
# Append mode: move cursor to the end of existing text
|
||||
await asyncio.to_thread(
|
||||
self.device.shell,
|
||||
"input keyevent 123", # KEYCODE_MOVE_END
|
||||
)
|
||||
|
||||
# 1. Fast path for ASCII text: Use ADB input text directly.
|
||||
if text.isascii():
|
||||
lines = text.split("\n")
|
||||
for i, line in enumerate(lines):
|
||||
if i > 0:
|
||||
# Send Enter key between lines
|
||||
await asyncio.to_thread(self.device.shell, "input keyevent 66")
|
||||
if line:
|
||||
escaped = _escape_for_adb_text(line)
|
||||
await asyncio.to_thread(self.device.shell, f"input text {escaped}")
|
||||
return True
|
||||
|
||||
# 2. For non-ASCII / Unicode text: Try UIAutomator client
|
||||
if self.ui_client and hasattr(self.ui_client, "send_text"):
|
||||
try:
|
||||
res = self.ui_client.send_text(text)
|
||||
if asyncio.iscoroutine(res):
|
||||
await res
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.warning(f"UIAutomator send_text failed: {e}")
|
||||
|
||||
# 3. Check if ADBKeyboard is currently active
|
||||
try:
|
||||
default_ime = await asyncio.to_thread(
|
||||
self.device.shell, "settings get secure default_input_method"
|
||||
)
|
||||
if "adbkeyboard" in str(default_ime).lower():
|
||||
b64_text = base64.b64encode(text.encode("utf-8")).decode("utf-8")
|
||||
broadcast_cmd = f"am broadcast -a ADB_INPUT_B64 --es msg '{b64_text}'"
|
||||
await asyncio.to_thread(self.device.shell, broadcast_cmd)
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 4. Fallback: ADB input text
|
||||
lines = text.split("\n")
|
||||
for i, line in enumerate(lines):
|
||||
if i > 0:
|
||||
await asyncio.to_thread(self.device.shell, "input keyevent 66")
|
||||
if line:
|
||||
escaped = _escape_for_adb_text(line)
|
||||
await asyncio.to_thread(self.device.shell, f"input text {escaped}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to type text '{text}': {e}")
|
||||
return False
|
||||
@@ -0,0 +1,57 @@
|
||||
# Copyright 2026 Google LLC
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Android screen recording and video transcode utilities."""
|
||||
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from artemis.utils.logger import get_logger
|
||||
|
||||
logger = get_logger(__name__)
|
||||
|
||||
|
||||
class ScreenRecorder:
|
||||
"""Manages background screen recording sessions using adb screenrecord."""
|
||||
|
||||
def __init__(self, adb_device: Any, output_path: Path):
|
||||
self.device = adb_device
|
||||
self.output_path = output_path
|
||||
self.is_recording = False
|
||||
self._remote_path = "/sdcard/artemis_record.mp4"
|
||||
|
||||
async def start(self) -> None:
|
||||
"""Starts remote screen recording."""
|
||||
self.is_recording = True
|
||||
logger.info("Starting screen recording on device...")
|
||||
# Start background screenrecord
|
||||
asyncio.create_task(
|
||||
asyncio.to_thread(self.device.shell, f"screenrecord {self._remote_path}")
|
||||
)
|
||||
|
||||
async def stop(self) -> Path | None:
|
||||
"""Stops screen recording and pulls file to host."""
|
||||
if not self.is_recording:
|
||||
return None
|
||||
self.is_recording = False
|
||||
logger.info("Stopping screen recording...")
|
||||
try:
|
||||
await asyncio.to_thread(self.device.shell, "pkill -2 screenrecord")
|
||||
await asyncio.sleep(1.0)
|
||||
self.output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
await asyncio.to_thread(self.device.sync.pull, self._remote_path, str(self.output_path))
|
||||
return self.output_path
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to pull screen recording: {e}")
|
||||
return None
|
||||
Reference in new issue
Block a user