feat(accessibility): implement Artemis Accessibility Helper for robust UI hierarchy capture

- Introduce ArtemisAccessibilityHelper APK (API 24 to 37+) as a conflict-free, high-performance replacement for UiAutomator:
  * Eliminates native UiAutomator waitForIdle() timeouts, reducing DOM dump latency from ~3s to ~15ms and maintaining resilience against infinite animations.
  * Implements a multi-tier fallback hierarchy discovery strategy (multi-window layer enumeration -> active window fallback -> input/accessibility focus backtracking) with adaptive progressive backoff for seamless page transition capture.
  * Resolves node drop defect caused by 32-bit hash collisions on recurring virtual view IDs in long lists and WebViews by utilizing bounded recursion (MAX_DEPTH=75, MAX_NODES=8000).
  * Strict W3C XML 1.0 character sanitization filtering invalid ASCII control codes and escaping attribute whitespace (
, 
, 	) to preserve multiline text formatting.
  * Enriches semantic parsing with stateDescription (API 30+ Jetpack Compose), screenReaderFocusable, paneTitle, heading, error, and tooltip attributes.
  * Multi-window metadata propagation tagging each node with window-id, window-type, and window-layer.
  * Adds dual-path standalone build script (build_apk.sh) producing a clean 29KB APK without Gradle daemon overhead alongside standard Gradle Kotlin DSL.
- Integrate AccessibilityClient into Artemis Python runtime with automatic bundled APK detection, self-healing installation, and port forwarding.
- Add unit test suite covering AccessibilityClient endpoints and atomic snapshot parsing.
This commit is contained in:
somew1nd committed 2026-09-10 14:10:23 -07:00
1 parent 38a2de6a9f
commit bac31b44c1
20 files changed
+2323

No files matched your search

+247
View File
@@ -0,0 +1,247 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Artemis Accessibility Helper client for Android device automation.
Provides a lightweight, conflict-free alternative to UIAutomator2.
Runs concurrently with Mobly, Appium, and Espresso without monopolizing
the singleton UiAutomationConnection.
"""
from io import BytesIO
import json
import os
import subprocess
import time
import urllib.error
import urllib.request
from PIL import Image
from artemis.clients.ui_automator_client import UIAutomatorScreenData, _pil_to_base64
from artemis.runtime.adb_endpoint import adb_command
from artemis.utils.logger import get_logger
logger = get_logger(__name__)
PACKAGE_NAME = "com.artemis.helper"
SERVICE_NAME = f"{PACKAGE_NAME}/.ArtemisAccessibilityService"
DEFAULT_PORT = 18888
class AccessibilityClient:
"""Non-exclusive accessibility client for Artemis UI automation."""
def __init__(self, device_id: str, local_port: int = DEFAULT_PORT):
self._device_id = device_id
self._local_port = local_port
self._base_url = f"http://127.0.0.1:{local_port}"
def ensure_service_ready(self, apk_path: str | None = None) -> bool:
"""Ensure ArtemisAccessibilityHelper is installed, enabled, and port-forwarded."""
# 1. Check if package is installed
if not self._is_installed():
if apk_path is None:
default_apk = os.path.abspath(
os.path.join(
os.path.dirname(__file__),
"../../packages/artemis-accessibility-helper/ArtemisAccessibilityHelper.apk",
)
)
if os.path.exists(default_apk):
apk_path = default_apk
if apk_path and os.path.exists(apk_path):
logger.info(f"Installing ArtemisAccessibilityHelper on {self._device_id}...")
self._run_adb(["install", "-r", "-g", apk_path])
else:
logger.warning(
f"ArtemisAccessibilityHelper not installed on {self._device_id} and no APK provided."
)
# 2. Ensure accessibility service is enabled via secure settings
self._enable_accessibility_service()
# 3. Setup port forward
self._setup_port_forward()
# 4. Verify connection
for attempt in range(5):
if self.ping():
logger.info(f"ArtemisAccessibilityHelper connected successfully on {self._device_id}")
return True
time.sleep(0.5)
logger.warning(f"Failed to connect to ArtemisAccessibilityHelper on port {self._local_port}")
return False
def ping(self) -> bool:
"""Health check probe."""
try:
req = urllib.request.Request(f"{self._base_url}/ping")
with urllib.request.urlopen(req, timeout=2.0) as resp:
data = json.loads(resp.read().decode())
return data.get("success", False)
except Exception:
return False
def get_screenshot(self) -> Image.Image | None:
"""Capture screenshot via high-speed ADB exec-out."""
try:
result = subprocess.run(
adb_command(["-s", self._device_id, "exec-out", "screencap", "-p"]),
stdin=subprocess.DEVNULL,
capture_output=True,
timeout=10,
check=True,
)
return Image.open(BytesIO(result.stdout))
except Exception as e:
logger.error(f"Failed to capture screenshot via adb: {e}")
return None
def get_hierarchy(self) -> dict:
"""Fetch hierarchy JSON directly from ArtemisAccessibilityHelper."""
req = urllib.request.Request(f"{self._base_url}/dump")
with urllib.request.urlopen(req, timeout=5.0) as resp:
return json.loads(resp.read().decode())
def get_hierarchy_xml(self) -> str:
"""Fetch raw standard UIAutomator XML string directly from helper."""
req = urllib.request.Request(f"{self._base_url}/dump_xml")
with urllib.request.urlopen(req, timeout=5.0) as resp:
return resp.read().decode("utf-8")
def get_atomic_snapshot(self) -> dict | None:
"""Fetch atomic snapshot (screenshot Base64 + hierarchy) directly in one call."""
try:
req = urllib.request.Request(f"{self._base_url}/snapshot")
with urllib.request.urlopen(req, timeout=6.0) as resp:
data = json.loads(resp.read().decode())
if data.get("success") and data.get("has_screenshot") and data.get("screenshot_base64"):
return data
except Exception as e:
logger.debug(f"Atomic snapshot request failed (falling back to dual path): {e}")
return None
def get_screen_data(self) -> UIAutomatorScreenData:
"""Retrieve complete screen data (screenshot + parsed elements + standard XML).
Prioritizes the atomic hardware snapshot API (Android 11+) to eliminate
any temporal phase mismatch between visual frame and layout DOM tree.
Falls back to separate ADB screencap + HTTP dump on older platforms.
"""
# 1. Try atomic snapshot
atomic = self.get_atomic_snapshot()
if atomic is not None:
return UIAutomatorScreenData(
base64=atomic.get("screenshot_base64", ""),
hierarchy_xml=atomic.get("xml", ""),
elements=atomic.get("elements", []),
width=atomic.get("width", 1080),
height=atomic.get("height", 2400),
)
# 2. Fallback: separate screenshot and hierarchy dump
screenshot = self.get_screenshot()
if screenshot is None:
raise RuntimeError("Failed to capture screenshot")
hierarchy_data = self.get_hierarchy()
elements = hierarchy_data.get("elements", [])
hierarchy_xml = hierarchy_data.get("xml", "")
return UIAutomatorScreenData(
base64=_pil_to_base64(screenshot, format="JPEG"),
hierarchy_xml=hierarchy_xml,
elements=elements,
width=screenshot.width,
height=screenshot.height,
)
def tap(self, x: float, y: float) -> bool:
"""Perform tap gesture via AccessibilityService."""
return self._send_rpc("tap", {"x": x, "y": y})
def swipe(self, x1: float, y1: float, x2: float, y2: float, duration_ms: int = 300) -> bool:
"""Perform swipe gesture via AccessibilityService."""
return self._send_rpc(
"swipe",
{"x1": x1, "y1": y1, "x2": x2, "y2": y2, "duration": duration_ms},
)
def send_text(self, text: str) -> bool:
"""Send text input via AccessibilityNodeInfo.ACTION_SET_TEXT."""
success = self._send_rpc("type", {"text": text})
if not success:
# Fallback to adb shell input text
safe_text = text.replace(" ", "%s")
self._run_adb(["shell", "input", "text", safe_text])
return True
return success
def clear_text(self) -> bool:
"""Clear text of focused input."""
return self._send_rpc("clear", {})
def press_key(self, key: str) -> bool:
"""Perform global navigation action (back, home, recents)."""
key_lower = key.lower()
if key_lower in ("back", "home", "recents", "notifications", "quick_settings"):
return self._send_rpc("global", {"action": key_lower})
# Fallback to adb keyevent
self._run_adb(["shell", "input", "keyevent", f"KEYCODE_{key.upper()}"])
return True
def _send_rpc(self, cmd: str, params: dict) -> bool:
"""Send JSON-RPC payload to helper server."""
payload = {"cmd": cmd, **params}
try:
req = urllib.request.Request(
f"{self._base_url}/action",
data=json.dumps(payload).encode("utf-8"),
headers={"Content-Type": "application/json"},
)
with urllib.request.urlopen(req, timeout=5.0) as resp:
data = json.loads(resp.read().decode())
return data.get("success", False)
except Exception as e:
logger.warning(f"RPC command '{cmd}' failed: {e}")
return False
def _is_installed(self) -> bool:
res = self._run_adb(["shell", "pm", "list", "packages", PACKAGE_NAME])
return f"package:{PACKAGE_NAME}" in res.stdout
def _enable_accessibility_service(self) -> None:
"""Silently enable the accessibility service via adb settings without UI prompts."""
res = self._run_adb(["shell", "settings", "get", "secure", "enabled_accessibility_services"])
current_services = res.stdout.strip()
if SERVICE_NAME not in current_services:
new_services = f"{current_services}:{SERVICE_NAME}" if current_services and current_services != "null" else SERVICE_NAME
self._run_adb(["shell", "settings", "put", "secure", "enabled_accessibility_services", new_services])
self._run_adb(["shell", "settings", "put", "secure", "accessibility_enabled", "1"])
def _setup_port_forward(self) -> None:
self._run_adb(["forward", f"tcp:{self._local_port}", f"tcp:{DEFAULT_PORT}"])
def _run_adb(self, args: list[str]) -> subprocess.CompletedProcess:
return subprocess.run(
adb_command(["-s", self._device_id] + args),
stdin=subprocess.DEVNULL,
capture_output=True,
text=True,
timeout=15,
check=False,
)