Files
Artemis-Eapil/artemis/drivers/android/adb_driver.py

580 lines
24 KiB
Python

# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Concrete Android Device Driver implementation using ADB and UIAutomator2."""
import asyncio
import base64
from io import BytesIO
from pathlib import Path
from typing import Any, Literal
from PIL import Image
from adbutils import AdbClient, AdbDevice
from artemis.clients.ui_automator_client import (
UIAutomatorClient,
_parse_hierarchy_xml_to_elements,
)
from artemis.config.agent import load_agent_config
from artemis.config.settings import settings
from artemis.config.paths import get_temp_dir
from artemis.drivers.base import BaseDeviceDriver, KeyCode, ScreenData, SwipeDirection
from artemis.drivers.android.app_launch import launch_android_app
from artemis.toolchain import find_ffmpeg, find_scrcpy
from artemis.utils.video import build_scrcpy_record_command
from artemis.utils.ui_filter import filter_ui_hierarchy
from artemis.utils.logger import get_logger
from artemis.utils.screenshot_limit import limit_screenshot_long_edge
logger = get_logger(__name__)
# KeyCode mapping from string name to Android keycode integer
ANDROID_KEYCODE_MAP: dict[str, int] = {
"home": 3,
"back": 4,
"enter": 66,
"delete": 67,
"power": 26,
"app_switch": 187,
"volume_up": 24,
"volume_down": 25,
}
def _escape_for_adb_text(s: str) -> str:
"""Escapes special characters for adb shell input text."""
return (
s.replace("\\", "\\\\")
.replace('"', '\\"')
.replace("'", "\\'")
.replace("`", "\\`")
.replace("$", "\\$")
.replace("&", "\\&")
.replace("|", "\\|")
.replace(";", "\\;")
.replace("<", "\\<")
.replace(">", "\\>")
.replace("(", "\\(")
.replace(")", "\\)")
.replace("*", "\\*")
.replace("?", "\\?")
.replace("~", "\\~")
.replace(" ", "%s")
)
class AndroidAdbDriver(BaseDeviceDriver):
"""Android device driver using ADB and UIAutomator2."""
def __init__(
self,
device_id: str,
adb_client: AdbClient,
ui_adb_client: UIAutomatorClient | None = None,
width: int = 1080,
height: int = 2400,
):
self._device_id = device_id
self._adb_client = adb_client
self._ui_adb_client = ui_adb_client
self._width = width
self._height = height
self._device: AdbDevice | None = None
self._recording_process: asyncio.subprocess.Process | None = None
self._recording_output_path: Path | None = None
self._screenshot_config = load_agent_config().screenshot
self._screen_capture_task: asyncio.Task[ScreenData] | None = None
@property
def device_id(self) -> str:
return self._device_id
@property
def screen_size(self) -> tuple[int, int]:
return (self._width, self._height)
@property
def device(self) -> AdbDevice:
if self._device is None:
self._device = self._adb_client.device(serial=self._device_id)
return self._device
async def connect(self) -> None:
"""Verify device availability."""
try:
state = self.device.get_state()
logger.info(f"Android device '{self._device_id}' connected (state: {state})")
except Exception as e:
logger.warning(f"Connecting to Android device '{self._device_id}': {e}")
async def disconnect(self) -> None:
"""Clean up ongoing recordings and tunnels."""
if self._recording_process:
await self.stop_video_recording()
if self._ui_adb_client and hasattr(self._ui_adb_client, "disconnect"):
await asyncio.to_thread(self._ui_adb_client.disconnect)
async def get_screen_data(self, skip_settling: bool = False) -> ScreenData:
"""Bound observation latency and share unfinished capture work across retries."""
task = self._screen_capture_task
if task is None or task.done():
task = asyncio.create_task(self._get_screen_data(skip_settling))
# A timed-out caller does not cancel a blocking SDK thread or start
# another concurrent read on the same device. Retrieve late errors.
task.add_done_callback(lambda done: None if done.cancelled() else done.exception())
self._screen_capture_task = task
try:
return await asyncio.wait_for(
asyncio.shield(task), timeout=settings.ARTEMIS_OBSERVATION_TIMEOUT_SECONDS
)
except TimeoutError as exc:
raise TimeoutError(
f"设备屏幕观测超时({settings.ARTEMIS_OBSERVATION_TIMEOUT_SECONDS:g} 秒)"
) from exc
async def _get_screen_data(self, skip_settling: bool = False) -> ScreenData:
"""Captures screenshot and XML hierarchy concurrently."""
if not skip_settling:
await asyncio.sleep(0.3)
# 1. Capture raw screenshot
screenshot_bytes: bytes = b""
screenshot_base64: str = ""
ui_hierarchy_xml: str | None = None
ui_elements: list[dict[str, Any]] = []
capture_errors: list[str] = []
# Check if ui_adb_client has direct screen data support
if self._ui_adb_client and hasattr(self._ui_adb_client, "get_screen_data"):
try:
raw_ui_data = await asyncio.to_thread(self._ui_adb_client.get_screen_data)
if asyncio.iscoroutine(raw_ui_data):
raw_ui_data = await raw_ui_data
if (
hasattr(raw_ui_data, "base64")
and isinstance(raw_ui_data.base64, str)
and raw_ui_data.base64
):
screenshot_base64 = raw_ui_data.base64
screenshot_bytes = base64.b64decode(screenshot_base64)
if hasattr(raw_ui_data, "width") and isinstance(raw_ui_data.width, int):
self._width = raw_ui_data.width
if hasattr(raw_ui_data, "height") and isinstance(raw_ui_data.height, int):
self._height = raw_ui_data.height
if hasattr(raw_ui_data, "hierarchy_xml") and isinstance(
raw_ui_data.hierarchy_xml, str
):
ui_hierarchy_xml = raw_ui_data.hierarchy_xml
if hasattr(raw_ui_data, "elements"):
raw_elements = raw_ui_data.elements
if isinstance(raw_elements, dict):
ui_elements = [raw_elements]
elif isinstance(raw_elements, list):
ui_elements = raw_elements
except Exception as e:
capture_errors.append(str(e))
logger.debug(f"Direct ui_adb_client.get_screen_data error: {e}")
if not screenshot_base64 or not isinstance(screenshot_base64, str):
try:
raw_image = await asyncio.to_thread(
self.device.shell,
["screencap", "-p"],
encoding=None,
timeout=settings.ARTEMIS_DEVICE_COMMAND_TIMEOUT_SECONDS,
)
pil_img = Image.open(BytesIO(raw_image))
if hasattr(pil_img, "save"):
buf = BytesIO()
pil_img.save(buf, format="PNG")
screenshot_bytes = buf.getvalue()
self._width, self._height = pil_img.size
screenshot_base64 = base64.b64encode(screenshot_bytes).decode("utf-8")
except Exception as e:
capture_errors.append(str(e))
logger.warning(f"Device screenshot capture failed on {self._device_id}: {e}")
if not screenshot_base64 or not isinstance(screenshot_base64, str):
raise RuntimeError(
"设备截图失败:ADB 与屏幕辅助服务均未返回有效截图;" + "; ".join(capture_errors)
)
with Image.open(BytesIO(screenshot_bytes)) as image:
if min(image.size) <= 1:
raise RuntimeError("设备截图无效:收到占位图,禁止基于旧画面继续操作")
image.verify()
# 2. Capture UI Hierarchy via UIAutomator
if self._ui_adb_client and not ui_elements:
try:
if hasattr(self._ui_adb_client, "get_hierarchy"):
res = await asyncio.to_thread(self._ui_adb_client.get_hierarchy)
hierarchy = await res if asyncio.iscoroutine(res) else res
if isinstance(hierarchy, str):
ui_hierarchy_xml = hierarchy
ui_elements = _parse_hierarchy_xml_to_elements(hierarchy)
else:
ui_elements = hierarchy
elif hasattr(self._ui_adb_client, "get_ui_elements"):
res = await asyncio.to_thread(self._ui_adb_client.get_ui_elements)
ui_elements = await res if asyncio.iscoroutine(res) else res
except Exception as e:
logger.debug(f"UI hierarchy extraction failed: {e}")
if isinstance(ui_elements, dict):
clean_ui_elements = [ui_elements]
elif isinstance(ui_elements, list):
clean_ui_elements = ui_elements
else:
clean_ui_elements = []
w = self._width if isinstance(self._width, int) else 1080
h = self._height if isinstance(self._height, int) else 2400
clean_ui_elements = filter_ui_hierarchy(
clean_ui_elements,
screen_width=w,
screen_height=h,
)
clean_b64 = screenshot_base64
clean_bytes = screenshot_bytes
if self._screenshot_config.enabled:
limited_bytes = limit_screenshot_long_edge(
clean_bytes, self._screenshot_config.max_long_edge
)
if limited_bytes is not clean_bytes:
clean_bytes = limited_bytes
clean_b64 = base64.b64encode(clean_bytes).decode("utf-8")
return ScreenData(
screenshot_bytes=clean_bytes,
screenshot_base64=clean_b64,
ui_hierarchy_xml=ui_hierarchy_xml,
ui_elements=clean_ui_elements,
width=w,
height=h,
platform="android",
)
async def tap(
self,
x: int,
y: int,
duration_ms: int = 100,
times: int = 1,
delay_ms: int = 100,
) -> bool:
try:
if duration_ms >= 500:
cmd = f"input swipe {x} {y} {x} {y} {duration_ms}"
else:
if times <= 1:
cmd = f"input tap {x} {y}"
else:
taps = [f"input tap {x} {y}"] * times
cmd = f" && sleep {delay_ms / 1000.0:.3f} && ".join(taps)
logger.info(f"[ADB] {cmd}")
await asyncio.to_thread(
self.device.shell, cmd, timeout=settings.ARTEMIS_DEVICE_COMMAND_TIMEOUT_SECONDS
)
return True
except Exception as e:
logger.error(f"Tap failed at ({x}, {y}): {e}")
return False
async def long_press(self, x: int, y: int, duration_ms: int = 1000) -> bool:
return await self.tap(x=x, y=y, duration_ms=duration_ms)
async def swipe(
self,
start_x: int,
start_y: int,
end_x: int,
end_y: int,
duration_ms: int = 800,
) -> bool:
try:
cmd = f"input swipe {start_x} {start_y} {end_x} {end_y} {duration_ms}"
logger.info(f"[ADB] {cmd}")
await asyncio.to_thread(
self.device.shell, cmd, timeout=settings.ARTEMIS_DEVICE_COMMAND_TIMEOUT_SECONDS
)
return True
except Exception as e:
logger.error(f"Swipe failed from ({start_x},{start_y}) to ({end_x},{end_y}): {e}")
return False
async def swipe_direction(
self,
direction: SwipeDirection | Literal["up", "down", "left", "right"],
duration_ms: int = 800,
) -> bool:
dir_str = str(direction).lower()
mid_x = int(
self._width * 0.6
) # Lock to 60% width to avoid edge gestures and alphabet fast-scroll sidebar
mid_y = self._height // 2
if dir_str == "up":
# Drag bottom to top -> scrolls down (0.7 -> 0.3 leaves ~50-60% overlap, 800ms prevents fling)
return await self.swipe(
mid_x, int(self._height * 0.7), mid_x, int(self._height * 0.3), duration_ms
)
elif dir_str == "down":
# Drag top to bottom -> scrolls up
return await self.swipe(
mid_x, int(self._height * 0.3), mid_x, int(self._height * 0.7), duration_ms
)
elif dir_str == "left":
# Drag right to left -> scrolls right
return await self.swipe(
int(self._width * 0.75), mid_y, int(self._width * 0.25), mid_y, duration_ms
)
elif dir_str == "right":
# Drag left to right -> scrolls left
return await self.swipe(
int(self._width * 0.25), mid_y, int(self._width * 0.75), mid_y, duration_ms
)
return False
async def input_text(self, text: str, clear_existing: bool = True) -> bool:
try:
if clear_existing:
# Safe & robust clearing: Move to End -> Shift+Home selection -> Delete -> Fallback backspaces
clear_cmd = (
"input keyevent 123 && "
"input keyevent --meta 1 122 && "
"input keyevent 67 && "
"input keyevent 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67 67"
)
await asyncio.to_thread(
self.device.shell,
clear_cmd,
timeout=settings.ARTEMIS_DEVICE_COMMAND_TIMEOUT_SECONDS,
)
else:
# Append mode: move cursor to the very end of existing text
await asyncio.to_thread(
self.device.shell,
"input keyevent 123", # KEYCODE_MOVE_END
)
# Normalize literal escaped newlines from LLM / tool call serialization
norm_text = text.replace(r"\r\n", "\n").replace(r"\n", "\n").replace(r"\r", "\n")
# 1. Tier 1: Try clipboard injection + KEYCODE_PASTE (Zero IME interference, preserves multiline, works for all charsets)
if self._ui_adb_client:
try:
set_clip_ok = False
if hasattr(self._ui_adb_client, "set_clipboard"):
set_clip_ok = self._ui_adb_client.set_clipboard(norm_text)
elif hasattr(self._ui_adb_client, "_device") and self._ui_adb_client._device:
self._ui_adb_client._device.set_clipboard(norm_text)
set_clip_ok = True
if set_clip_ok:
await asyncio.to_thread(
self.device.shell,
"input keyevent 279",
timeout=settings.ARTEMIS_DEVICE_COMMAND_TIMEOUT_SECONDS,
)
return True
except Exception as e:
logger.debug(f"Clipboard paste fallback to ADB input: {e}")
# 2. Tier 2: Check if ADBKeyboard is currently active
try:
default_ime = await asyncio.to_thread(
self.device.shell, "settings get secure default_input_method"
)
if "adbkeyboard" in str(default_ime).lower():
b64_text = base64.b64encode(norm_text.encode("utf-8")).decode("utf-8")
broadcast_cmd = f"am broadcast -a ADB_INPUT_B64 --es msg '{b64_text}'"
await asyncio.to_thread(
self.device.shell,
broadcast_cmd,
timeout=settings.ARTEMIS_DEVICE_COMMAND_TIMEOUT_SECONDS,
)
return True
except Exception as e:
# ADBKeyboard probe/broadcast failed; fall through to native input.
logger.debug(f"ADBKeyboard IME path failed, falling back to ADB input: {e}")
# 3. Tier 3: Universal Native ADB input text fallback
lines = norm_text.split("\n")
for i, line in enumerate(lines):
if i > 0:
# Send Enter key between lines
await asyncio.to_thread(
self.device.shell,
"input keyevent 66",
timeout=settings.ARTEMIS_DEVICE_COMMAND_TIMEOUT_SECONDS,
)
if line:
escaped = _escape_for_adb_text(line)
await asyncio.to_thread(
self.device.shell,
f"input text {escaped}",
timeout=settings.ARTEMIS_DEVICE_COMMAND_TIMEOUT_SECONDS,
)
return True
except Exception as e:
logger.error(f"Input text failed for '{text}': {e}")
return False
async def press_key(self, key: KeyCode | str | int) -> bool:
try:
keycode_val = key
if isinstance(key, (KeyCode, str)):
key_name = str(key).lower().replace("keycode.", "")
keycode_val = ANDROID_KEYCODE_MAP.get(key_name, key)
await asyncio.to_thread(
self.device.shell,
f"input keyevent {keycode_val}",
timeout=settings.ARTEMIS_DEVICE_COMMAND_TIMEOUT_SECONDS,
)
return True
except Exception as e:
logger.error(f"Press key failed for '{key}': {e}")
return False
async def launch_app(self, package_name: str) -> bool:
return await launch_android_app(
self.device, package_name, settings.ARTEMIS_DEVICE_COMMAND_TIMEOUT_SECONDS
)
async def stop_app(self, package_name: str) -> bool:
try:
await asyncio.to_thread(
self.device.shell,
f"am force-stop {package_name}",
timeout=settings.ARTEMIS_DEVICE_COMMAND_TIMEOUT_SECONDS,
)
return True
except Exception as e:
logger.error(f"Stop app failed for '{package_name}': {e}")
return False
async def get_current_package(self) -> str | None:
try:
# 1. Try modern current_app from adbutils
if hasattr(self.device, "current_app"):
try:
app_info = await asyncio.to_thread(self.device.current_app)
if app_info and getattr(app_info, "package", None):
return app_info.package
except Exception as e:
# current_app is unsupported on some devices; use dumpsys below.
logger.debug(f"current_app query failed, falling back to dumpsys: {e}")
# 2. Structured dumpsys extraction
out = await asyncio.to_thread(
self.device.shell, "dumpsys window displays | grep -E 'mCurrentFocus|mFocusedApp'"
)
for line in str(out).splitlines():
if "/" in line:
for token in line.split():
if "/" in token and "." in token:
clean = token.split("/")[0].strip("{} ,")
if clean and not clean.startswith("Window") and "." in clean:
return clean
except Exception as e:
logger.debug(f"Error querying current package: {e}")
return None
async def execute_shell(self, command: str, timeout_seconds: float = 15.0) -> str:
try:
return await asyncio.wait_for(
asyncio.to_thread(
self.device.shell,
command,
timeout=settings.ARTEMIS_DEVICE_COMMAND_TIMEOUT_SECONDS,
),
timeout=timeout_seconds,
)
except Exception as e:
return f"Error: {e}"
async def start_video_recording(self, output_dir: Path | None = None) -> None:
"""Starts screen recording via scrcpy in background."""
out_dir = output_dir or get_temp_dir("recordings")
out_dir.mkdir(parents=True, exist_ok=True)
self._recording_mkv_path = out_dir / "recording.mkv"
self._recording_output_path = out_dir / "recording.mp4"
logger.info(f"Starting scrcpy video recording to {self._recording_mkv_path}...")
try:
scrcpy_bin = find_scrcpy()
cmd = build_scrcpy_record_command(
scrcpy_bin,
self.device_id,
self._recording_mkv_path,
lock_capture_orientation=False,
)
self._scrcpy_process = await asyncio.create_subprocess_exec(
*cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
await asyncio.sleep(0.5)
except Exception as e:
logger.warning(f"Failed to start scrcpy subprocess: {e}")
async def stop_video_recording(self) -> str | None:
"""Stops background video capture and returns local recording file path."""
if hasattr(self, "_scrcpy_process") and self._scrcpy_process:
try:
self._scrcpy_process.terminate()
await asyncio.wait_for(self._scrcpy_process.wait(), timeout=5.0)
except (TimeoutError, ProcessLookupError, OSError) as e:
# Already exited, or did not stop within the timeout; a lingering
# scrcpy may keep the MKV file locked on Windows.
logger.debug(f"scrcpy process did not terminate cleanly: {e}")
self._scrcpy_process = None
mkv = getattr(self, "_recording_mkv_path", None)
mp4 = getattr(self, "_recording_output_path", None)
if mkv and mkv.exists() and mp4:
try:
proc = await asyncio.create_subprocess_exec(
find_ffmpeg(),
"-y",
"-i",
str(mkv),
"-c",
"copy",
"-movflags",
"+faststart",
str(mp4),
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
await proc.wait()
if mp4.exists() and mp4.stat().st_size > 0:
try:
mkv.unlink()
except OSError:
# Best-effort cleanup of the intermediate MKV file.
pass
return str(mp4)
except Exception as e:
logger.warning(f"Failed to convert MKV to MP4 in AdbDriver: {e}")
if mp4 and mp4.exists():
return str(mp4)
if mkv and mkv.exists():
return str(mkv)
return None