feat(accessibility): implement Artemis Accessibility Helper for robust UI hierarchy capture

- Introduce ArtemisAccessibilityHelper APK (API 24 to 37+) as a conflict-free, high-performance replacement for UiAutomator:
  * Eliminates native UiAutomator waitForIdle() timeouts, reducing DOM dump latency from ~3s to ~15ms and maintaining resilience against infinite animations.
  * Implements a multi-tier fallback hierarchy discovery strategy (multi-window layer enumeration -> active window fallback -> input/accessibility focus backtracking) with adaptive progressive backoff for seamless page transition capture.
  * Resolves node drop defect caused by 32-bit hash collisions on recurring virtual view IDs in long lists and WebViews by utilizing bounded recursion (MAX_DEPTH=75, MAX_NODES=8000).
  * Strict W3C XML 1.0 character sanitization filtering invalid ASCII control codes and escaping attribute whitespace (
, 
, 	) to preserve multiline text formatting.
  * Enriches semantic parsing with stateDescription (API 30+ Jetpack Compose), screenReaderFocusable, paneTitle, heading, error, and tooltip attributes.
  * Multi-window metadata propagation tagging each node with window-id, window-type, and window-layer.
  * Adds dual-path standalone build script (build_apk.sh) producing a clean 29KB APK without Gradle daemon overhead alongside standard Gradle Kotlin DSL.
- Integrate AccessibilityClient into Artemis Python runtime with automatic bundled APK detection, self-healing installation, and port forwarding.
- Add unit test suite covering AccessibilityClient endpoints and atomic snapshot parsing.
This commit is contained in:
somew1nd committed 2026-09-10 14:10:23 -07:00
1 parent 38a2de6a9f
commit bac31b44c1
20 files changed
+2323

No files matched your search

+247
View File
@@ -0,0 +1,247 @@
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Artemis Accessibility Helper client for Android device automation.
Provides a lightweight, conflict-free alternative to UIAutomator2.
Runs concurrently with Mobly, Appium, and Espresso without monopolizing
the singleton UiAutomationConnection.
"""
from io import BytesIO
import json
import os
import subprocess
import time
import urllib.error
import urllib.request
from PIL import Image
from artemis.clients.ui_automator_client import UIAutomatorScreenData, _pil_to_base64
from artemis.runtime.adb_endpoint import adb_command
from artemis.utils.logger import get_logger
logger = get_logger(__name__)
PACKAGE_NAME = "com.artemis.helper"
SERVICE_NAME = f"{PACKAGE_NAME}/.ArtemisAccessibilityService"
DEFAULT_PORT = 18888
class AccessibilityClient:
"""Non-exclusive accessibility client for Artemis UI automation."""
def __init__(self, device_id: str, local_port: int = DEFAULT_PORT):
self._device_id = device_id
self._local_port = local_port
self._base_url = f"http://127.0.0.1:{local_port}"
def ensure_service_ready(self, apk_path: str | None = None) -> bool:
"""Ensure ArtemisAccessibilityHelper is installed, enabled, and port-forwarded."""
# 1. Check if package is installed
if not self._is_installed():
if apk_path is None:
default_apk = os.path.abspath(
os.path.join(
os.path.dirname(__file__),
"../../packages/artemis-accessibility-helper/ArtemisAccessibilityHelper.apk",
)
)
if os.path.exists(default_apk):
apk_path = default_apk
if apk_path and os.path.exists(apk_path):
logger.info(f"Installing ArtemisAccessibilityHelper on {self._device_id}...")
self._run_adb(["install", "-r", "-g", apk_path])
else:
logger.warning(
f"ArtemisAccessibilityHelper not installed on {self._device_id} and no APK provided."
)
# 2. Ensure accessibility service is enabled via secure settings
self._enable_accessibility_service()
# 3. Setup port forward
self._setup_port_forward()
# 4. Verify connection
for attempt in range(5):
if self.ping():
logger.info(f"ArtemisAccessibilityHelper connected successfully on {self._device_id}")
return True
time.sleep(0.5)
logger.warning(f"Failed to connect to ArtemisAccessibilityHelper on port {self._local_port}")
return False
def ping(self) -> bool:
"""Health check probe."""
try:
req = urllib.request.Request(f"{self._base_url}/ping")
with urllib.request.urlopen(req, timeout=2.0) as resp:
data = json.loads(resp.read().decode())
return data.get("success", False)
except Exception:
return False
def get_screenshot(self) -> Image.Image | None:
"""Capture screenshot via high-speed ADB exec-out."""
try:
result = subprocess.run(
adb_command(["-s", self._device_id, "exec-out", "screencap", "-p"]),
stdin=subprocess.DEVNULL,
capture_output=True,
timeout=10,
check=True,
)
return Image.open(BytesIO(result.stdout))
except Exception as e:
logger.error(f"Failed to capture screenshot via adb: {e}")
return None
def get_hierarchy(self) -> dict:
"""Fetch hierarchy JSON directly from ArtemisAccessibilityHelper."""
req = urllib.request.Request(f"{self._base_url}/dump")
with urllib.request.urlopen(req, timeout=5.0) as resp:
return json.loads(resp.read().decode())
def get_hierarchy_xml(self) -> str:
"""Fetch raw standard UIAutomator XML string directly from helper."""
req = urllib.request.Request(f"{self._base_url}/dump_xml")
with urllib.request.urlopen(req, timeout=5.0) as resp:
return resp.read().decode("utf-8")
def get_atomic_snapshot(self) -> dict | None:
"""Fetch atomic snapshot (screenshot Base64 + hierarchy) directly in one call."""
try:
req = urllib.request.Request(f"{self._base_url}/snapshot")
with urllib.request.urlopen(req, timeout=6.0) as resp:
data = json.loads(resp.read().decode())
if data.get("success") and data.get("has_screenshot") and data.get("screenshot_base64"):
return data
except Exception as e:
logger.debug(f"Atomic snapshot request failed (falling back to dual path): {e}")
return None
def get_screen_data(self) -> UIAutomatorScreenData:
"""Retrieve complete screen data (screenshot + parsed elements + standard XML).
Prioritizes the atomic hardware snapshot API (Android 11+) to eliminate
any temporal phase mismatch between visual frame and layout DOM tree.
Falls back to separate ADB screencap + HTTP dump on older platforms.
"""
# 1. Try atomic snapshot
atomic = self.get_atomic_snapshot()
if atomic is not None:
return UIAutomatorScreenData(
base64=atomic.get("screenshot_base64", ""),
hierarchy_xml=atomic.get("xml", ""),
elements=atomic.get("elements", []),
width=atomic.get("width", 1080),
height=atomic.get("height", 2400),
)
# 2. Fallback: separate screenshot and hierarchy dump
screenshot = self.get_screenshot()
if screenshot is None:
raise RuntimeError("Failed to capture screenshot")
hierarchy_data = self.get_hierarchy()
elements = hierarchy_data.get("elements", [])
hierarchy_xml = hierarchy_data.get("xml", "")
return UIAutomatorScreenData(
base64=_pil_to_base64(screenshot, format="JPEG"),
hierarchy_xml=hierarchy_xml,
elements=elements,
width=screenshot.width,
height=screenshot.height,
)
def tap(self, x: float, y: float) -> bool:
"""Perform tap gesture via AccessibilityService."""
return self._send_rpc("tap", {"x": x, "y": y})
def swipe(self, x1: float, y1: float, x2: float, y2: float, duration_ms: int = 300) -> bool:
"""Perform swipe gesture via AccessibilityService."""
return self._send_rpc(
"swipe",
{"x1": x1, "y1": y1, "x2": x2, "y2": y2, "duration": duration_ms},
)
def send_text(self, text: str) -> bool:
"""Send text input via AccessibilityNodeInfo.ACTION_SET_TEXT."""
success = self._send_rpc("type", {"text": text})
if not success:
# Fallback to adb shell input text
safe_text = text.replace(" ", "%s")
self._run_adb(["shell", "input", "text", safe_text])
return True
return success
def clear_text(self) -> bool:
"""Clear text of focused input."""
return self._send_rpc("clear", {})
def press_key(self, key: str) -> bool:
"""Perform global navigation action (back, home, recents)."""
key_lower = key.lower()
if key_lower in ("back", "home", "recents", "notifications", "quick_settings"):
return self._send_rpc("global", {"action": key_lower})
# Fallback to adb keyevent
self._run_adb(["shell", "input", "keyevent", f"KEYCODE_{key.upper()}"])
return True
def _send_rpc(self, cmd: str, params: dict) -> bool:
"""Send JSON-RPC payload to helper server."""
payload = {"cmd": cmd, **params}
try:
req = urllib.request.Request(
f"{self._base_url}/action",
data=json.dumps(payload).encode("utf-8"),
headers={"Content-Type": "application/json"},
)
with urllib.request.urlopen(req, timeout=5.0) as resp:
data = json.loads(resp.read().decode())
return data.get("success", False)
except Exception as e:
logger.warning(f"RPC command '{cmd}' failed: {e}")
return False
def _is_installed(self) -> bool:
res = self._run_adb(["shell", "pm", "list", "packages", PACKAGE_NAME])
return f"package:{PACKAGE_NAME}" in res.stdout
def _enable_accessibility_service(self) -> None:
"""Silently enable the accessibility service via adb settings without UI prompts."""
res = self._run_adb(["shell", "settings", "get", "secure", "enabled_accessibility_services"])
current_services = res.stdout.strip()
if SERVICE_NAME not in current_services:
new_services = f"{current_services}:{SERVICE_NAME}" if current_services and current_services != "null" else SERVICE_NAME
self._run_adb(["shell", "settings", "put", "secure", "enabled_accessibility_services", new_services])
self._run_adb(["shell", "settings", "put", "secure", "accessibility_enabled", "1"])
def _setup_port_forward(self) -> None:
self._run_adb(["forward", f"tcp:{self._local_port}", f"tcp:{DEFAULT_PORT}"])
def _run_adb(self, args: list[str]) -> subprocess.CompletedProcess:
return subprocess.run(
adb_command(["-s", self._device_id] + args),
stdin=subprocess.DEVNULL,
capture_output=True,
text=True,
timeout=15,
check=False,
)
@@ -0,0 +1,5 @@
build/
*.idsig
.gradle/
*.class
.DS_Store
@@ -0,0 +1,35 @@
plugins {
id("com.android.application")
}
android {
namespace = "com.artemis.helper"
compileSdk = 35
defaultConfig {
applicationId = "com.artemis.helper"
minSdk = 24
targetSdk = 35
versionCode = 1
versionName = "1.0.0"
}
buildTypes {
release {
isMinifyEnabled = false
proguardFiles(
getDefaultProguardFile("proguard-android-optimize.txt"),
"proguard-rules.pro"
)
}
}
compileOptions {
sourceCompatibility = JavaVersion.VERSION_1_8
targetCompatibility = JavaVersion.VERSION_1_8
}
}
dependencies {
// Pure standard Android SDK APIs (android.accessibilityservice, android.view.accessibility, org.json)
// Zero external dependencies to maximize stability, guarantee compatibility, and keep APK under 30KB.
}
@@ -0,0 +1,37 @@
<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
package="com.artemis.helper">
<uses-sdk
android:minSdkVersion="24"
android:targetSdkVersion="35" />
<uses-permission android:name="android.permission.INTERNET" />
<uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_SPECIAL_USE" />
<uses-permission android:name="android.permission.POST_NOTIFICATIONS" />
<application
android:allowBackup="false"
android:label="@string/app_name"
android:supportsRtl="true">
<service
android:name=".ArtemisAccessibilityService"
android:permission="android.permission.BIND_ACCESSIBILITY_SERVICE"
android:exported="true"
android:foregroundServiceType="specialUse">
<intent-filter>
<action android:name="android.accessibilityservice.AccessibilityService" />
</intent-filter>
<meta-data
android:name="android.accessibilityservice"
android:resource="@xml/accessibility_service_config" />
<property
android:name="android.app.PROPERTY_SPECIAL_USE_FGS_SUBTYPE"
android:value="Artemis mobile automation test bridge" />
</service>
</application>
</manifest>
@@ -0,0 +1,264 @@
package com.artemis.helper;
import android.graphics.Rect;
import org.json.JSONArray;
import org.json.JSONObject;
import java.util.ArrayList;
import java.util.List;
/**
* In-memory snapshot of an Accessibility UI element node.
* Decoupled from live Android AccessibilityNodeInfo objects to prevent
* Binder proxy memory leaks, avoid stale node crashes, and ensure
* instant thread-safe serialization to both XML and JSON.
*/
public final class A11yNode {
public int index = 0;
public String text = "";
public String resourceId = "";
public String className = "";
public String packageName = "";
public String contentDesc = "";
public boolean checkable = false;
public boolean checked = false;
public boolean clickable = false;
public boolean enabled = true;
public boolean focusable = false;
public boolean focused = false;
public boolean scrollable = false;
public boolean longClickable = false;
public boolean password = false;
public boolean selected = false;
// Window metadata for multi-window awareness
public int windowId = -1;
public String windowType = "";
public int windowLayer = 0;
public boolean windowActive = false;
public boolean windowFocused = false;
// Advanced semantics for Agent perception & verification
public boolean editable = false;
public boolean isHeading = false;
public boolean screenReaderFocusable = false;
public String stateDescription = "";
public String errorText = "";
public String paneTitle = "";
public String tooltip = "";
public int left = 0;
public int top = 0;
public int right = 0;
public int bottom = 0;
public int drawingOrder = 0;
public String hint = "";
public final List<A11yNode> children = new ArrayList<>(4);
public A11yNode() {}
public int getWidth() {
return right - left;
}
public int getHeight() {
return bottom - top;
}
public String getBoundsString() {
return "[" + left + "," + top + "][" + right + "," + bottom + "]";
}
/**
* Serializes this node and all of its descendants into standard UIAutomator XML.
*/
public void writeXml(StringBuilder sb) {
sb.append("<node");
XmlUtils.appendIntAttribute(sb, "index", index);
XmlUtils.appendAttribute(sb, "text", text);
XmlUtils.appendAttribute(sb, "resource-id", resourceId);
XmlUtils.appendAttribute(sb, "class", className);
XmlUtils.appendAttribute(sb, "package", packageName);
XmlUtils.appendAttribute(sb, "content-desc", contentDesc);
XmlUtils.appendBooleanAttribute(sb, "checkable", checkable);
XmlUtils.appendBooleanAttribute(sb, "checked", checked);
XmlUtils.appendBooleanAttribute(sb, "clickable", clickable);
XmlUtils.appendBooleanAttribute(sb, "enabled", enabled);
XmlUtils.appendBooleanAttribute(sb, "focusable", focusable);
XmlUtils.appendBooleanAttribute(sb, "focused", focused);
XmlUtils.appendBooleanAttribute(sb, "scrollable", scrollable);
XmlUtils.appendBooleanAttribute(sb, "long-clickable", longClickable);
XmlUtils.appendBooleanAttribute(sb, "password", password);
XmlUtils.appendBooleanAttribute(sb, "selected", selected);
XmlUtils.appendAttribute(sb, "bounds", getBoundsString());
XmlUtils.appendIntAttribute(sb, "drawing-order", drawingOrder);
XmlUtils.appendAttribute(sb, "hint", hint);
// Window metadata (emitted on window root nodes)
if (windowId >= 0) {
XmlUtils.appendIntAttribute(sb, "window-id", windowId);
if (!windowType.isEmpty()) {
XmlUtils.appendAttribute(sb, "window-type", windowType);
}
XmlUtils.appendIntAttribute(sb, "window-layer", windowLayer);
if (windowActive) {
XmlUtils.appendBooleanAttribute(sb, "window-active", true);
}
if (windowFocused) {
XmlUtils.appendBooleanAttribute(sb, "window-focused", true);
}
}
// Extended semantic attributes
if (editable) {
XmlUtils.appendBooleanAttribute(sb, "editable", true);
}
if (isHeading) {
XmlUtils.appendBooleanAttribute(sb, "heading", true);
}
if (screenReaderFocusable) {
XmlUtils.appendBooleanAttribute(sb, "screen-reader-focusable", true);
}
if (!stateDescription.isEmpty()) {
XmlUtils.appendAttribute(sb, "state-description", stateDescription);
}
if (!errorText.isEmpty()) {
XmlUtils.appendAttribute(sb, "error", errorText);
}
if (!paneTitle.isEmpty()) {
XmlUtils.appendAttribute(sb, "pane-title", paneTitle);
}
if (!tooltip.isEmpty()) {
XmlUtils.appendAttribute(sb, "tooltip", tooltip);
}
if (children.isEmpty()) {
sb.append(" />");
} else {
sb.append('>');
for (int i = 0; i < children.size(); i++) {
children.get(i).writeXml(sb);
}
sb.append("</node>");
}
}
/**
* Serializes this node into a hierarchical JSON tree object.
*/
public JSONObject toTreeJson() {
JSONObject obj = toBaseJson();
try {
if (!children.isEmpty()) {
JSONArray childrenArr = new JSONArray();
for (int i = 0; i < children.size(); i++) {
childrenArr.put(children.get(i).toTreeJson());
}
obj.put("children", childrenArr);
}
} catch (Throwable ignored) {}
return obj;
}
/**
* Serializes this node into an element JSON object for the flat list.
* Does NOT include child references.
*/
public JSONObject toFlatElementJson() {
return toBaseJson();
}
private JSONObject toBaseJson() {
JSONObject obj = new JSONObject();
try {
obj.put("class", className);
obj.put("package", packageName);
obj.put("resource-id", resourceId);
obj.put("text", text);
obj.put("content-desc", contentDesc);
obj.put("bounds", getBoundsString());
JSONObject bounds = new JSONObject();
bounds.put("left", left);
bounds.put("top", top);
bounds.put("right", right);
bounds.put("bottom", bottom);
obj.put("parsed_bounds", bounds);
obj.put("clickable", clickable);
obj.put("scrollable", scrollable);
obj.put("checkable", checkable);
obj.put("checked", checked);
obj.put("enabled", enabled);
obj.put("focusable", focusable);
obj.put("focused", focused);
obj.put("selected", selected);
obj.put("password", password);
obj.put("long-clickable", longClickable);
obj.put("drawing-order", drawingOrder);
if (!hint.isEmpty()) {
obj.put("hint", hint);
}
// Window metadata
if (windowId >= 0) {
obj.put("window_id", windowId);
obj.put("window_type", windowType);
obj.put("window_layer", windowLayer);
if (windowActive) obj.put("window_active", true);
if (windowFocused) obj.put("window_focused", true);
}
// Extended semantics
if (editable) {
obj.put("editable", true);
}
if (isHeading) {
obj.put("is_heading", true);
}
if (screenReaderFocusable) {
obj.put("screen_reader_focusable", true);
}
if (!stateDescription.isEmpty()) {
obj.put("state_description", stateDescription);
}
if (!errorText.isEmpty()) {
obj.put("error", errorText);
}
if (!paneTitle.isEmpty()) {
obj.put("pane_title", paneTitle);
}
if (!tooltip.isEmpty()) {
obj.put("tooltip", tooltip);
}
} catch (Throwable ignored) {}
return obj;
}
/**
* Determines whether this node contains useful information or interactivity
* and should be included in the flat elements array for agent perception.
*/
public boolean isInformativeOrInteractive() {
boolean hasContent = !text.isEmpty() || !contentDesc.isEmpty() || !resourceId.isEmpty()
|| !stateDescription.isEmpty() || !errorText.isEmpty() || !paneTitle.isEmpty();
boolean isInteractive = clickable || scrollable || checkable || focusable || longClickable || editable;
return hasContent || isInteractive || isHeading;
}
/**
* Recursively collects all informative or interactive nodes into a flat list.
*/
public void collectFlatElements(List<JSONObject> flatList) {
if (isInformativeOrInteractive()) {
flatList.add(toFlatElementJson());
}
for (int i = 0; i < children.size(); i++) {
children.get(i).collectFlatElements(flatList);
}
}
}
@@ -0,0 +1,172 @@
package com.artemis.helper;
import android.accessibilityservice.AccessibilityService;
import android.accessibilityservice.AccessibilityServiceInfo;
import android.app.Notification;
import android.app.NotificationChannel;
import android.app.NotificationManager;
import android.content.Context;
import android.content.Intent;
import android.content.pm.ServiceInfo;
import android.os.Build;
import android.util.Log;
import android.view.accessibility.AccessibilityEvent;
/**
* Artemis non-exclusive Accessibility Service with foreground keep-alive support.
*
* Runs concurrently with any Android UI testing framework (Mobly, Appium, Espresso)
* without monopolizing Android's singleton UiAutomationService connection.
*/
public class ArtemisAccessibilityService extends AccessibilityService {
private static final String TAG = "ArtemisA11yService";
public static final int DEFAULT_PORT = 18888;
private static final String CHANNEL_ID = "artemis_helper_channel";
private static final int NOTIFICATION_ID = 18888;
private static volatile ArtemisAccessibilityService instance;
private volatile String currentPackageName = "";
private volatile String currentActivityName = "";
private CommandServer server;
public static ArtemisAccessibilityService getInstance() {
return instance;
}
public String getCurrentPackageName() {
return currentPackageName;
}
public String getCurrentActivityName() {
return currentActivityName;
}
@Override
public void onServiceConnected() {
super.onServiceConnected();
instance = this;
Log.i(TAG, "ArtemisAccessibilityService connected");
// 1. Start foreground service to resist low-memory killer on aggressive custom ROMs
startForegroundNotification();
// 2. Dynamically enforce flags to guarantee compatibility across custom OEM ROMs
try {
AccessibilityServiceInfo info = getServiceInfo();
if (info == null) {
info = new AccessibilityServiceInfo();
}
info.eventTypes = AccessibilityEvent.TYPES_ALL_MASK;
info.feedbackType = AccessibilityServiceInfo.FEEDBACK_GENERIC;
info.notificationTimeout = 50;
info.flags |= AccessibilityServiceInfo.FLAG_RETRIEVE_INTERACTIVE_WINDOWS
| AccessibilityServiceInfo.FLAG_INCLUDE_NOT_IMPORTANT_VIEWS
| AccessibilityServiceInfo.FLAG_REPORT_VIEW_IDS;
setServiceInfo(info);
Log.i(TAG, "AccessibilityServiceInfo flags dynamically enforced");
} catch (Throwable t) {
Log.w(TAG, "Failed to dynamically configure AccessibilityServiceInfo", t);
}
// 3. Start local loopback command server
if (server != null) {
server.shutdown();
server = null;
}
server = new CommandServer(this, DEFAULT_PORT);
server.setDaemon(true);
server.start();
Log.i(TAG, "CommandServer started on port " + DEFAULT_PORT);
}
@Override
public int onStartCommand(Intent intent, int flags, int startId) {
return START_STICKY;
}
@SuppressWarnings("deprecation")
private void startForegroundNotification() {
try {
NotificationManager nm = (NotificationManager) getSystemService(Context.NOTIFICATION_SERVICE);
if (nm == null) return;
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
NotificationChannel channel = new NotificationChannel(
CHANNEL_ID,
"Artemis Helper Service",
NotificationManager.IMPORTANCE_LOW
);
channel.setDescription("Keeps Artemis Accessibility Helper active for automation testing");
channel.setShowBadge(false);
nm.createNotificationChannel(channel);
}
Notification.Builder builder;
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
builder = new Notification.Builder(this, CHANNEL_ID);
} else {
builder = new Notification.Builder(this);
}
builder.setContentTitle("Artemis Accessibility Helper")
.setContentText("Active on 127.0.0.1:" + DEFAULT_PORT)
.setSmallIcon(android.R.drawable.stat_notify_sync)
.setOngoing(true);
if (Build.VERSION.SDK_INT >= 34) {
startForeground(NOTIFICATION_ID, builder.build(), ServiceInfo.FOREGROUND_SERVICE_TYPE_SPECIAL_USE);
} else if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
startForeground(NOTIFICATION_ID, builder.build(), 0);
} else {
startForeground(NOTIFICATION_ID, builder.build());
}
Log.i(TAG, "Foreground keep-alive notification active");
} catch (Throwable t) {
Log.w(TAG, "Foreground notification start skipped or deferred: " + t.getMessage());
}
}
@SuppressWarnings("deprecation")
private void stopForegroundNotification() {
try {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
stopForeground(STOP_FOREGROUND_REMOVE);
} else {
stopForeground(true);
}
} catch (Throwable ignored) {}
}
@Override
public void onAccessibilityEvent(AccessibilityEvent event) {
if (event == null) return;
if (event.getEventType() == AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED) {
CharSequence pkg = event.getPackageName();
CharSequence cls = event.getClassName();
if (pkg != null) currentPackageName = pkg.toString();
if (cls != null) currentActivityName = cls.toString();
}
}
@Override
public void onInterrupt() {
Log.w(TAG, "ArtemisAccessibilityService interrupted");
}
@Override
public void onDestroy() {
super.onDestroy();
Log.i(TAG, "ArtemisAccessibilityService destroyed");
stopForegroundNotification();
if (server != null) {
server.shutdown();
server = null;
}
if (instance == this) {
instance = null;
}
}
}
@@ -0,0 +1,302 @@
package com.artemis.helper;
import android.util.Log;
import org.json.JSONObject;
import java.io.BufferedReader;
import java.io.Closeable;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.net.Socket;
import java.nio.charset.StandardCharsets;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
/**
* High-performance, multi-threaded command server bound strictly to loopback (127.0.0.1:18888).
* Supports:
* - HTTP REST API (/dump, /dump_xml, /hierarchy.xml, /ping, /action)
* - Line-delimited JSON-RPC over TCP socket
*/
public class CommandServer extends Thread {
private static final String TAG = "ArtemisCommandServer";
private final ArtemisAccessibilityService service;
private final int port;
private volatile boolean isRunning = true;
private ServerSocket serverSocket;
private final ExecutorService clientExecutor = Executors.newCachedThreadPool();
public CommandServer(ArtemisAccessibilityService service, int port) {
super("ArtemisCommandServer");
this.service = service;
this.port = port;
}
@Override
public void run() {
try {
serverSocket = new ServerSocket();
serverSocket.setReuseAddress(true);
serverSocket.bind(new InetSocketAddress(InetAddress.getByName("127.0.0.1"), port), 50);
Log.i(TAG, "CommandServer listening on 127.0.0.1:" + port);
while (isRunning) {
final Socket socket;
try {
socket = serverSocket.accept();
} catch (Exception e) {
if (!isRunning) break;
continue;
}
clientExecutor.execute(new Runnable() {
@Override
public void run() {
handleClient(socket);
}
});
}
} catch (Exception e) {
Log.e(TAG, "Server error", e);
} finally {
closeQuietly(serverSocket);
}
}
private void handleClient(Socket socket) {
try {
socket.setSoTimeout(10000);
BufferedReader reader = new BufferedReader(new InputStreamReader(socket.getInputStream(), StandardCharsets.UTF_8));
OutputStream output = socket.getOutputStream();
String firstLine = reader.readLine();
if (firstLine == null) return;
String trimmed = firstLine.trim();
if (trimmed.startsWith("GET ") || trimmed.startsWith("POST ")) {
handleHttp(trimmed, reader, output);
} else if (trimmed.startsWith("{")) {
handleJsonRpc(trimmed, output);
} else {
JSONObject err = new JSONObject();
err.put("success", false);
err.put("error", "Unsupported protocol");
sendJsonResponse(output, err.toString());
}
} catch (Exception e) {
Log.w(TAG, "Client handling error: " + e.getMessage());
} finally {
closeQuietly(socket);
}
}
private void handleHttp(String requestLine, BufferedReader reader, OutputStream output) {
try {
String[] parts = requestLine.split(" ");
String path = parts.length > 1 ? parts[1] : "/";
int contentLength = 0;
String line;
while ((line = reader.readLine()) != null) {
if (line.isEmpty()) break;
String lower = line.toLowerCase();
if (lower.startsWith("content-length:")) {
try {
contentLength = Integer.parseInt(lower.substring("content-length:".length()).trim());
} catch (Exception ignored) {}
}
}
String body = "";
if (contentLength > 0) {
char[] buf = new char[contentLength];
int total = 0;
while (total < contentLength) {
int r = reader.read(buf, total, contentLength - total);
if (r < 0) break;
total += r;
}
body = new String(buf, 0, total);
}
// Route endpoints
if (path.startsWith("/snapshot")) {
JSONObject json = HierarchyDumper.dumpAtomicSnapshot(service);
sendJsonResponse(output, json.toString());
} else if (path.equals("/dump_xml") || path.equals("/hierarchy.xml") || path.startsWith("/dump?format=xml")) {
String xml = HierarchyDumper.dumpXml(service);
sendXmlResponse(output, xml);
} else if (path.startsWith("/dump") || path.startsWith("/hierarchy")) {
JSONObject json = HierarchyDumper.dump(service);
sendJsonResponse(output, json.toString());
} else if (path.startsWith("/ping") || path.equals("/")) {
JSONObject r = new JSONObject();
r.put("success", true);
r.put("service", "ArtemisAccessibilityService");
r.put("package", service.getCurrentPackageName());
r.put("activity", service.getCurrentActivityName());
sendJsonResponse(output, r.toString());
} else if (path.startsWith("/action") || path.startsWith("/rpc")) {
JSONObject json = body.isEmpty() ? new JSONObject() : new JSONObject(body);
JSONObject resp = executeCommand(json.optString("cmd", ""), json);
sendJsonResponse(output, resp.toString());
} else {
JSONObject r = new JSONObject();
r.put("success", false);
r.put("error", "Unknown endpoint: " + path);
sendJsonResponse(output, r.toString());
}
} catch (Exception e) {
Log.e(TAG, "HTTP error", e);
}
}
private void handleJsonRpc(String line, OutputStream output) {
try {
JSONObject json = new JSONObject(line);
String cmd = json.optString("cmd", "");
JSONObject resp = executeCommand(cmd, json);
byte[] bytes = resp.toString().getBytes(StandardCharsets.UTF_8);
output.write(bytes);
output.write('\n');
output.flush();
} catch (Exception e) {
try {
JSONObject err = new JSONObject();
err.put("success", false);
err.put("error", "JSON parse error: " + e.getMessage());
output.write(err.toString().getBytes(StandardCharsets.UTF_8));
output.write('\n');
output.flush();
} catch (Exception ignored) {}
}
}
private JSONObject executeCommand(String cmd, JSONObject params) {
JSONObject resp = new JSONObject();
try {
switch (cmd.toLowerCase()) {
case "ping":
resp.put("success", true);
resp.put("service", "ArtemisAccessibilityService");
resp.put("package", service.getCurrentPackageName());
resp.put("activity", service.getCurrentActivityName());
break;
case "dump":
case "dump_ui":
return HierarchyDumper.dump(service);
case "snapshot":
return HierarchyDumper.dumpAtomicSnapshot(service);
case "dump_xml":
resp.put("success", true);
resp.put("xml", HierarchyDumper.dumpXml(service));
break;
case "tap":
float x = (float) params.optDouble("x", -1.0);
float y = (float) params.optDouble("y", -1.0);
long tapTimeout = params.optLong("timeout", 1500L);
if (x < 0 || y < 0) {
resp.put("success", false);
resp.put("error", "Invalid coordinates");
} else {
resp.put("success", GestureController.tap(service, x, y, tapTimeout));
}
break;
case "double_tap":
float dtx = (float) params.optDouble("x", -1.0);
float dty = (float) params.optDouble("y", -1.0);
long dtTimeout = params.optLong("timeout", 2000L);
if (dtx < 0 || dty < 0) {
resp.put("success", false);
resp.put("error", "Invalid coordinates");
} else {
resp.put("success", GestureController.doubleTap(service, dtx, dty, dtTimeout));
}
break;
case "long_press":
float lpx = (float) params.optDouble("x", -1.0);
float lpy = (float) params.optDouble("y", -1.0);
long lpDuration = params.optLong("duration", 1000L);
if (lpx < 0 || lpy < 0) {
resp.put("success", false);
resp.put("error", "Invalid coordinates");
} else {
resp.put("success", GestureController.longPress(service, lpx, lpy, lpDuration, 2500L));
}
break;
case "swipe":
float x1 = (float) params.optDouble("x1", -1.0);
float y1 = (float) params.optDouble("y1", -1.0);
float x2 = (float) params.optDouble("x2", -1.0);
float y2 = (float) params.optDouble("y2", -1.0);
long dur = params.optLong("duration", 300L);
resp.put("success", GestureController.swipe(service, x1, y1, x2, y2, dur, 3000L));
break;
case "type":
String text = params.optString("text", "");
resp.put("success", GestureController.setText(service, text));
break;
case "clear":
resp.put("success", GestureController.clearText(service));
break;
case "global":
String action = params.optString("action", "");
resp.put("success", GestureController.performGlobalAction(service, action));
break;
default:
resp.put("success", false);
resp.put("error", "Unknown command: " + cmd);
break;
}
} catch (Exception e) {
try {
resp.put("success", false);
resp.put("error", e.getMessage());
} catch (Exception ignored) {}
}
return resp;
}
private void sendJsonResponse(OutputStream output, String jsonStr) {
try {
byte[] bytes = jsonStr.getBytes(StandardCharsets.UTF_8);
String header = "HTTP/1.1 200 OK\r\n" +
"Content-Type: application/json; charset=utf-8\r\n" +
"Content-Length: " + bytes.length + "\r\n" +
"Connection: close\r\n\r\n";
output.write(header.getBytes(StandardCharsets.UTF_8));
output.write(bytes);
output.flush();
} catch (Exception ignored) {}
}
private void sendXmlResponse(OutputStream output, String xmlStr) {
try {
byte[] bytes = xmlStr.getBytes(StandardCharsets.UTF_8);
String header = "HTTP/1.1 200 OK\r\n" +
"Content-Type: application/xml; charset=utf-8\r\n" +
"Content-Length: " + bytes.length + "\r\n" +
"Connection: close\r\n\r\n";
output.write(header.getBytes(StandardCharsets.UTF_8));
output.write(bytes);
output.flush();
} catch (Exception ignored) {}
}
public void shutdown() {
isRunning = false;
closeQuietly(serverSocket);
clientExecutor.shutdownNow();
}
private static void closeQuietly(Closeable c) {
if (c != null) {
try { c.close(); } catch (Throwable ignored) {}
}
}
}
@@ -0,0 +1,111 @@
package com.artemis.helper;
import android.accessibilityservice.AccessibilityService;
import android.content.Context;
import android.graphics.Point;
import android.os.Build;
import android.util.DisplayMetrics;
import android.view.Display;
import android.view.Surface;
import android.view.WindowManager;
/**
* Universal display and orientation utilities compatible across all Android versions (API 24 - 37+).
*/
public final class DisplayUtils {
private DisplayUtils() {}
public static class DisplayInfo {
public final int rotation;
public final int width;
public final int height;
public DisplayInfo(int rotation, int width, int height) {
this.rotation = rotation;
this.width = width;
this.height = height;
}
}
/**
* Obtains the display orientation (0, 1, 2, 3) and screen dimensions (width, height)
* using the most reliable API for the current Android runtime.
*/
@SuppressWarnings("deprecation")
public static DisplayInfo getDisplayInfo(AccessibilityService service) {
int rotation = 0;
int width = 1080;
int height = 2400;
// Baseline initialization from resources metrics (handles tablets, TVs, emulators dynamically)
try {
DisplayMetrics resDm = service.getResources().getDisplayMetrics();
if (resDm != null && resDm.widthPixels > 0 && resDm.heightPixels > 0) {
width = resDm.widthPixels;
height = resDm.heightPixels;
}
} catch (Throwable ignored) {}
try {
// Modern API 30+ window metrics for full physical screen dimensions
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
try {
WindowManager wm = (WindowManager) service.getSystemService(Context.WINDOW_SERVICE);
if (wm != null) {
android.graphics.Rect bounds = wm.getMaximumWindowMetrics().getBounds();
if (bounds.width() > 0 && bounds.height() > 0) {
width = bounds.width();
height = bounds.height();
}
}
} catch (Throwable ignored) {}
}
// Resolve Display instance for rotation and legacy metrics
Display display = null;
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
try {
display = service.getDisplay();
} catch (Throwable ignored) {}
}
if (display == null) {
WindowManager wm = (WindowManager) service.getSystemService(Context.WINDOW_SERVICE);
if (wm != null) {
display = wm.getDefaultDisplay();
}
}
if (display != null) {
int r = display.getRotation();
switch (r) {
case Surface.ROTATION_0:
rotation = 0;
break;
case Surface.ROTATION_90:
rotation = 1;
break;
case Surface.ROTATION_180:
rotation = 2;
break;
case Surface.ROTATION_270:
rotation = 3;
break;
default:
rotation = 0;
break;
}
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) {
DisplayMetrics dm = new DisplayMetrics();
display.getRealMetrics(dm);
width = dm.widthPixels;
height = dm.heightPixels;
}
}
} catch (Throwable ignored) {}
return new DisplayInfo(rotation, width, height);
}
}
@@ -0,0 +1,174 @@
package com.artemis.helper;
import android.accessibilityservice.AccessibilityService;
import android.accessibilityservice.GestureDescription;
import android.graphics.Path;
import android.os.Build;
import android.os.Bundle;
import android.os.Handler;
import android.os.Looper;
import android.util.Log;
import android.view.accessibility.AccessibilityNodeInfo;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
/**
* Synchronous gesture and action executor using native AccessibilityService APIs.
* Works without requiring UiAutomation or root privileges.
*/
public final class GestureController {
private static final String TAG = "ArtemisGestureCtrl";
private static final Handler MAIN_HANDLER = new Handler(Looper.getMainLooper());
private GestureController() {}
public static boolean tap(AccessibilityService service, float x, float y, long timeoutMs) {
Path path = new Path();
path.moveTo(x, y);
GestureDescription.StrokeDescription stroke = new GestureDescription.StrokeDescription(path, 0L, 60L);
GestureDescription gesture = new GestureDescription.Builder().addStroke(stroke).build();
return dispatchSynchronous(service, gesture, timeoutMs);
}
public static boolean doubleTap(AccessibilityService service, float x, float y, long timeoutMs) {
if (!tap(service, x, y, timeoutMs)) return false;
try {
Thread.sleep(100L);
} catch (InterruptedException ignored) {}
return tap(service, x, y, timeoutMs);
}
public static boolean longPress(AccessibilityService service, float x, float y, long durationMs, long timeoutMs) {
Path path = new Path();
path.moveTo(x, y);
long pressDur = Math.max(500L, Math.min(durationMs, 5000L));
GestureDescription.StrokeDescription stroke = new GestureDescription.StrokeDescription(path, 0L, pressDur);
GestureDescription gesture = new GestureDescription.Builder().addStroke(stroke).build();
return dispatchSynchronous(service, gesture, timeoutMs + pressDur);
}
public static boolean swipe(
AccessibilityService service,
float x1, float y1, float x2, float y2,
long durationMs, long timeoutMs
) {
Path path = new Path();
path.moveTo(x1, y1);
path.lineTo(x2, y2);
long safeDuration = Math.max(50L, Math.min(durationMs, 5000L));
GestureDescription.StrokeDescription stroke = new GestureDescription.StrokeDescription(path, 0L, safeDuration);
GestureDescription gesture = new GestureDescription.Builder().addStroke(stroke).build();
return dispatchSynchronous(service, gesture, timeoutMs + safeDuration);
}
public static boolean setText(AccessibilityService service, String text) {
AccessibilityNodeInfo inputNode = HierarchyDumper.findInputNode(service);
if (inputNode == null) {
Log.w(TAG, "No editable/focused input node found for setText");
return false;
}
try {
Bundle args = new Bundle();
args.putCharSequence(AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, text);
return inputNode.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, args);
} catch (Throwable t) {
Log.w(TAG, "performAction ACTION_SET_TEXT failed", t);
return false;
} finally {
HierarchyDumper.safeRecycle(inputNode);
}
}
public static boolean clearText(AccessibilityService service) {
return setText(service, "");
}
public static boolean performGlobalAction(AccessibilityService service, String actionName) {
if (actionName == null) return false;
int actionId;
String lower = actionName.toLowerCase();
switch (lower) {
case "back":
actionId = AccessibilityService.GLOBAL_ACTION_BACK;
break;
case "home":
actionId = AccessibilityService.GLOBAL_ACTION_HOME;
break;
case "recents":
actionId = AccessibilityService.GLOBAL_ACTION_RECENTS;
break;
case "notifications":
actionId = AccessibilityService.GLOBAL_ACTION_NOTIFICATIONS;
break;
case "quick_settings":
actionId = AccessibilityService.GLOBAL_ACTION_QUICK_SETTINGS;
break;
case "power_dialog":
actionId = AccessibilityService.GLOBAL_ACTION_POWER_DIALOG;
break;
case "toggle_split_screen":
actionId = AccessibilityService.GLOBAL_ACTION_TOGGLE_SPLIT_SCREEN;
break;
case "lock_screen":
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) {
actionId = AccessibilityService.GLOBAL_ACTION_LOCK_SCREEN;
} else {
return false;
}
break;
case "take_screenshot":
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) {
actionId = AccessibilityService.GLOBAL_ACTION_TAKE_SCREENSHOT;
} else {
return false;
}
break;
default:
return false;
}
return service.performGlobalAction(actionId);
}
private static boolean dispatchSynchronous(
final AccessibilityService service,
final GestureDescription gesture,
long timeoutMs
) {
final CountDownLatch latch = new CountDownLatch(1);
final AtomicBoolean result = new AtomicBoolean(false);
MAIN_HANDLER.post(new Runnable() {
@Override
public void run() {
try {
service.dispatchGesture(gesture, new AccessibilityService.GestureResultCallback() {
@Override
public void onCompleted(GestureDescription gestureDescription) {
result.set(true);
latch.countDown();
}
@Override
public void onCancelled(GestureDescription gestureDescription) {
result.set(false);
latch.countDown();
}
}, null);
} catch (Throwable t) {
Log.w(TAG, "dispatchGesture threw exception", t);
result.set(false);
latch.countDown();
}
}
});
try {
return latch.await(timeoutMs, TimeUnit.MILLISECONDS) && result.get();
} catch (InterruptedException e) {
return false;
}
}
}
@@ -0,0 +1,632 @@
package com.artemis.helper;
import android.accessibilityservice.AccessibilityService;
import android.graphics.Bitmap;
import android.graphics.ColorSpace;
import android.graphics.Rect;
import android.hardware.HardwareBuffer;
import android.os.Build;
import android.os.SystemClock;
import android.util.Base64;
import android.util.Log;
import android.view.Display;
import android.view.accessibility.AccessibilityNodeInfo;
import android.view.accessibility.AccessibilityWindowInfo;
import org.json.JSONArray;
import org.json.JSONObject;
import java.io.ByteArrayOutputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
/**
* Universal, ultra-stable UI hierarchy dumper for Android (API 24 - 37+).
*
* Capabilities:
* 1. Zero WaitForIdle hangs: Bypasses UIAutomator's waitForIdle() timeout.
* 2. Multi-window penetration: Captures App, Dialog, System UI, IME Keyboard, Split-Screen.
* 3. Rich semantic extraction: Captures errorText, isHeading, editable, paneTitle, tooltip.
* 4. Atomic screenshot snapshot: On API 30+, captures hardware bitmap and DOM tree simultaneously.
* 5. W3C XML 1.0 compliance: Strict character sanitization prevents Python parse failures.
*/
public final class HierarchyDumper {
private static final String TAG = "ArtemisHierarchyDumper";
private static final int MAX_DEPTH = 75;
private static final int MAX_NODES = 8000;
private static final int RETRY_COUNT = 3;
private static final long RETRY_INTERVAL_MS = 40L;
private static final ExecutorService SCREENSHOT_EXECUTOR = Executors.newSingleThreadExecutor();
private HierarchyDumper() {}
/**
* Dumps the complete hierarchy as a JSON response.
*/
public static JSONObject dump(AccessibilityService service) {
long startTime = System.currentTimeMillis();
JSONObject result = new JSONObject();
JSONArray elementsJson = new JSONArray();
try {
DisplayUtils.DisplayInfo displayInfo = DisplayUtils.getDisplayInfo(service);
List<A11yNode> rootSnapshots = captureRootSnapshots(service);
if (rootSnapshots.isEmpty()) {
result.put("success", false);
result.put("error", "No active window or root node found");
result.put("xml", "");
result.put("elements", elementsJson);
result.put("rotation", displayInfo.rotation);
result.put("width", displayInfo.width);
result.put("height", displayInfo.height);
return result;
}
// 1. Build standard UIAutomator XML
String xml = buildXml(rootSnapshots, displayInfo.rotation, displayInfo.width, displayInfo.height);
// 2. Build flat elements list
List<JSONObject> flatList = new ArrayList<>();
for (A11yNode root : rootSnapshots) {
root.collectFlatElements(flatList);
}
for (JSONObject elem : flatList) {
elementsJson.put(elem);
}
// 3. Build tree
JSONArray trees = new JSONArray();
for (A11yNode root : rootSnapshots) {
trees.put(root.toTreeJson());
}
long elapsed = System.currentTimeMillis() - startTime;
result.put("success", true);
result.put("xml", xml);
result.put("elements", elementsJson);
result.put("tree", trees.length() == 1 ? trees.getJSONObject(0) : trees);
result.put("rotation", displayInfo.rotation);
result.put("width", displayInfo.width);
result.put("height", displayInfo.height);
result.put("elapsed_ms", elapsed);
if (service instanceof ArtemisAccessibilityService) {
ArtemisAccessibilityService s = (ArtemisAccessibilityService) service;
result.put("package", s.getCurrentPackageName());
result.put("activity", s.getCurrentActivityName());
}
} catch (Throwable t) {
Log.e(TAG, "Dump failed with exception", t);
try {
result.put("success", false);
result.put("error", "Dump failed: " + t.getMessage());
result.put("xml", "");
result.put("elements", elementsJson);
} catch (Throwable ignored) {}
}
return result;
}
/**
* Dumps an atomic snapshot combining hardware screenshot (JPEG base64) and UI hierarchy XML/JSON
* at the exact same clock tick, eliminating temporal phase mismatch.
*/
public static JSONObject dumpAtomicSnapshot(AccessibilityService service) {
long startTime = System.currentTimeMillis();
// 1. Trigger hardware screenshot asynchronously
final AtomicReference<Bitmap> bitmapRef = new AtomicReference<>(null);
final CountDownLatch screenshotLatch = new CountDownLatch(1);
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
try {
service.takeScreenshot(
Display.DEFAULT_DISPLAY,
SCREENSHOT_EXECUTOR,
new AccessibilityService.TakeScreenshotCallback() {
@Override
public void onSuccess(AccessibilityService.ScreenshotResult screenshotResult) {
try {
HardwareBuffer buffer = screenshotResult.getHardwareBuffer();
ColorSpace colorSpace = screenshotResult.getColorSpace();
Bitmap hwBitmap = Bitmap.wrapHardwareBuffer(buffer, colorSpace);
if (hwBitmap != null) {
Bitmap swBitmap = hwBitmap.copy(Bitmap.Config.ARGB_8888, false);
hwBitmap.recycle();
buffer.close();
bitmapRef.set(swBitmap);
}
} catch (Throwable t) {
Log.w(TAG, "Error copying screenshot buffer", t);
} finally {
screenshotLatch.countDown();
}
}
@Override
public void onFailure(int errorCode) {
Log.w(TAG, "takeScreenshot failed, errorCode: " + errorCode);
screenshotLatch.countDown();
}
}
);
} catch (Throwable t) {
Log.w(TAG, "takeScreenshot invocation error", t);
screenshotLatch.countDown();
}
} else {
screenshotLatch.countDown();
}
// 2. Concurrently capture the UI hierarchy
JSONObject dumpData = dump(service);
// 3. Wait for the screenshot with a strict 1.5s timeout
try {
screenshotLatch.await(1500L, TimeUnit.MILLISECONDS);
} catch (InterruptedException ignored) {}
Bitmap bitmap = bitmapRef.get();
if (bitmap != null) {
try {
ByteArrayOutputStream baos = new ByteArrayOutputStream(bitmap.getWidth() * bitmap.getHeight() / 4);
bitmap.compress(Bitmap.CompressFormat.JPEG, 80, baos);
byte[] jpegBytes = baos.toByteArray();
String base64Str = Base64.encodeToString(jpegBytes, Base64.NO_WRAP);
dumpData.put("screenshot_base64", base64Str);
dumpData.put("has_screenshot", true);
} catch (Throwable t) {
Log.w(TAG, "Failed to compress screenshot to JPEG Base64", t);
try { dumpData.put("has_screenshot", false); } catch (Throwable ignored) {}
} finally {
bitmap.recycle();
}
} else {
try {
dumpData.put("has_screenshot", false);
dumpData.put("screenshot_error", Build.VERSION.SDK_INT >= Build.VERSION_CODES.R
? "Screenshot capture timed out or failed"
: "takeScreenshot not supported on Android < 11");
} catch (Throwable ignored) {}
}
try {
dumpData.put("atomic_elapsed_ms", System.currentTimeMillis() - startTime);
} catch (Throwable ignored) {}
return dumpData;
}
/**
* Dumps the hierarchy directly as a raw standard UIAutomator XML string.
*/
public static String dumpXml(AccessibilityService service) {
DisplayUtils.DisplayInfo displayInfo = DisplayUtils.getDisplayInfo(service);
List<A11yNode> rootSnapshots = captureRootSnapshots(service);
if (rootSnapshots.isEmpty()) {
return "<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>\n<hierarchy rotation=\""
+ displayInfo.rotation + "\" />\n";
}
return buildXml(rootSnapshots, displayInfo.rotation, displayInfo.width, displayInfo.height);
}
/**
* Builds the standard Android UIAutomator XML string from root snapshots.
*/
private static String buildXml(List<A11yNode> roots, int rotation, int width, int height) {
StringBuilder sb = new StringBuilder(roots.size() * 1024 + 256);
sb.append("<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>\n");
sb.append("<hierarchy rotation=\"").append(rotation).append("\">");
for (int i = 0; i < roots.size(); i++) {
A11yNode root = roots.get(i);
root.index = i;
root.writeXml(sb);
}
sb.append("</hierarchy>\n");
return sb.toString();
}
private static final class RawRootEntry {
final AccessibilityNodeInfo root;
final int windowId;
final String windowType;
final int windowLayer;
final boolean windowActive;
final boolean windowFocused;
RawRootEntry(
AccessibilityNodeInfo root,
int windowId,
String windowType,
int windowLayer,
boolean windowActive,
boolean windowFocused
) {
this.root = root;
this.windowId = windowId;
this.windowType = windowType;
this.windowLayer = windowLayer;
this.windowActive = windowActive;
this.windowFocused = windowFocused;
}
}
/**
* Captures snapshots of all active and interactive windows with adaptive progressive retry support.
* Guaranteed to capture the complete hierarchy even during activity transitions and cold starts.
*/
public static List<A11yNode> captureRootSnapshots(AccessibilityService service) {
List<RawRootEntry> rawRoots = Collections.emptyList();
long[] retryBackoff = new long[]{40L, 80L, 120L, 160L, 220L, 300L};
for (int attempt = 0; attempt <= retryBackoff.length; attempt++) {
rawRoots = getActiveRawRoots(service);
if (!rawRoots.isEmpty()) {
break;
}
if (attempt < retryBackoff.length) {
SystemClock.sleep(retryBackoff[attempt]);
}
}
List<A11yNode> snapshots = new ArrayList<>(rawRoots.size());
int[] nodeCounter = new int[]{0};
for (int i = 0; i < rawRoots.size(); i++) {
RawRootEntry entry = rawRoots.get(i);
try {
A11yNode snapshot = snapshotNode(entry.root, 0, i, nodeCounter);
if (snapshot != null) {
snapshot.windowId = entry.windowId;
snapshot.windowType = entry.windowType;
snapshot.windowLayer = entry.windowLayer;
snapshot.windowActive = entry.windowActive;
snapshot.windowFocused = entry.windowFocused;
snapshots.add(snapshot);
}
} catch (Throwable t) {
Log.w(TAG, "Failed to snapshot root window " + entry.windowId, t);
} finally {
safeRecycle(entry.root);
}
}
return snapshots;
}
/**
* Multi-tier hierarchy root discovery strategy:
* Tier 1: Multi-window enumeration (sorted by Z-layer descending).
* Tier 2: Active window direct fallback (if getWindows() is empty or missing foreground app).
* Tier 3: Focused input / accessibility node backtracking (walks up parent chain to top root).
*/
private static List<RawRootEntry> getActiveRawRoots(AccessibilityService service) {
List<RawRootEntry> roots = new ArrayList<>();
Set<Integer> seenHashes = new HashSet<>();
boolean hasAppWindow = false;
// Tier 1: Multi-window enumeration (App, Dialogs, Popups, Keyboards, Split-screen)
try {
List<AccessibilityWindowInfo> windows = service.getWindows();
if (windows != null && !windows.isEmpty()) {
List<AccessibilityWindowInfo> sortedWindows = new ArrayList<>(windows);
Collections.sort(sortedWindows, new Comparator<AccessibilityWindowInfo>() {
@Override
public int compare(AccessibilityWindowInfo w1, AccessibilityWindowInfo w2) {
return Integer.compare(w2.getLayer(), w1.getLayer());
}
});
for (AccessibilityWindowInfo window : sortedWindows) {
try {
AccessibilityNodeInfo root = window.getRoot();
if (root != null) {
int hash = root.hashCode();
if (seenHashes.add(hash)) {
String typeStr = resolveWindowType(window.getType());
if (window.getType() == AccessibilityWindowInfo.TYPE_APPLICATION) {
hasAppWindow = true;
}
roots.add(new RawRootEntry(
root,
window.getId(),
typeStr,
window.getLayer(),
window.isActive(),
window.isFocused()
));
} else {
safeRecycle(root);
}
}
} catch (Throwable ignored) {}
}
}
} catch (Throwable ignored) {}
// Tier 2: Active window fallback (if getWindows() returned empty or lacked active app window)
if (!hasAppWindow) {
try {
AccessibilityNodeInfo activeRoot = service.getRootInActiveWindow();
if (activeRoot != null) {
int hash = activeRoot.hashCode();
if (seenHashes.add(hash)) {
roots.add(0, new RawRootEntry(
activeRoot,
activeRoot.getWindowId(),
"application",
0,
true,
true
));
hasAppWindow = true;
} else {
safeRecycle(activeRoot);
}
}
} catch (Throwable ignored) {}
}
// Tier 3: Focused node backtracking (recovers window tree during transient transitions)
if (roots.isEmpty()) {
AccessibilityNodeInfo focused = null;
try {
focused = service.findFocus(AccessibilityNodeInfo.FOCUS_INPUT);
} catch (Throwable ignored) {}
if (focused == null) {
try {
focused = service.findFocus(AccessibilityNodeInfo.FOCUS_ACCESSIBILITY);
} catch (Throwable ignored) {}
}
if (focused != null) {
try {
AccessibilityNodeInfo current = focused;
AccessibilityNodeInfo parent = current.getParent();
while (parent != null) {
if (current != focused) {
safeRecycle(current);
}
current = parent;
parent = current.getParent();
}
int hash = current.hashCode();
if (seenHashes.add(hash)) {
roots.add(new RawRootEntry(
current,
current.getWindowId(),
"application",
0,
true,
true
));
} else {
safeRecycle(current);
}
} catch (Throwable ignored) {
} finally {
safeRecycle(focused);
}
}
}
return roots;
}
private static String resolveWindowType(int type) {
switch (type) {
case AccessibilityWindowInfo.TYPE_APPLICATION:
return "application";
case AccessibilityWindowInfo.TYPE_INPUT_METHOD:
return "input_method";
case AccessibilityWindowInfo.TYPE_SYSTEM:
return "system";
case AccessibilityWindowInfo.TYPE_ACCESSIBILITY_OVERLAY:
return "accessibility_overlay";
case AccessibilityWindowInfo.TYPE_SPLIT_SCREEN_DIVIDER:
return "split_screen_divider";
default:
return "unknown";
}
}
/**
* Recursively snapshots an AccessibilityNodeInfo into an immutable A11yNode,
* extracting rich semantic properties (error, heading, editable, paneTitle, tooltip, stateDescription).
*
* Protected against recursion loops via MAX_DEPTH and MAX_NODES without flat hash collision drops.
*/
@SuppressWarnings("deprecation")
private static A11yNode snapshotNode(
AccessibilityNodeInfo node,
int depth,
int childIndex,
int[] nodeCounter
) {
if (node == null) return null;
if (depth > MAX_DEPTH || nodeCounter[0] >= MAX_NODES) {
return null;
}
nodeCounter[0]++;
A11yNode snapshot = new A11yNode();
snapshot.index = childIndex;
try {
Rect bounds = new Rect();
node.getBoundsInScreen(bounds);
snapshot.left = bounds.left;
snapshot.top = bounds.top;
snapshot.right = bounds.right;
snapshot.bottom = bounds.bottom;
CharSequence text = node.getText();
CharSequence desc = node.getContentDescription();
CharSequence pkg = node.getPackageName();
CharSequence cls = node.getClassName();
String resId = node.getViewIdResourceName();
snapshot.text = text != null ? text.toString() : "";
snapshot.contentDesc = desc != null ? desc.toString() : "";
snapshot.packageName = pkg != null ? pkg.toString() : "";
snapshot.className = cls != null ? cls.toString() : "";
snapshot.resourceId = resId != null ? resId : "";
snapshot.clickable = node.isClickable();
snapshot.checkable = node.isCheckable();
snapshot.checked = node.isChecked();
snapshot.enabled = node.isEnabled();
snapshot.focusable = node.isFocusable();
snapshot.focused = node.isFocused();
snapshot.scrollable = node.isScrollable();
snapshot.longClickable = node.isLongClickable();
snapshot.password = node.isPassword();
snapshot.selected = node.isSelected();
// 1. Editable property
snapshot.editable = node.isEditable();
// 2. Error message (crucial for form validation detection)
CharSequence err = node.getError();
if (err != null && err.length() > 0) {
snapshot.errorText = err.toString();
}
// 3. Drawing order (API 24+)
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
try {
snapshot.drawingOrder = node.getDrawingOrder();
} catch (Throwable ignored) {}
}
// 4. Hint text (API 26+)
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
try {
CharSequence hint = node.getHintText();
if (hint != null) snapshot.hint = hint.toString();
} catch (Throwable ignored) {}
}
// 5. Heading, PaneTitle, Tooltip, ScreenReaderFocusable (API 28+)
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) {
try {
snapshot.isHeading = node.isHeading();
} catch (Throwable ignored) {}
try {
snapshot.screenReaderFocusable = node.isScreenReaderFocusable();
} catch (Throwable ignored) {}
try {
CharSequence pt = node.getPaneTitle();
if (pt != null) snapshot.paneTitle = pt.toString();
} catch (Throwable ignored) {}
try {
CharSequence tt = node.getTooltipText();
if (tt != null) snapshot.tooltip = tt.toString();
} catch (Throwable ignored) {}
}
// 6. State Description (API 30+ Jetpack Compose semantics: expanded, collapsed, etc.)
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
try {
CharSequence sd = node.getStateDescription();
if (sd != null) snapshot.stateDescription = sd.toString();
} catch (Throwable ignored) {}
}
// Recursively process children
int childCount = node.getChildCount();
for (int i = 0; i < childCount; i++) {
AccessibilityNodeInfo childNode = null;
try {
childNode = node.getChild(i);
if (childNode != null) {
A11yNode childSnapshot = snapshotNode(childNode, depth + 1, i, nodeCounter);
if (childSnapshot != null) {
snapshot.children.add(childSnapshot);
}
}
} catch (Throwable ignored) {
} finally {
if (childNode != null) {
safeRecycle(childNode);
}
}
}
} catch (Throwable t) {
Log.w(TAG, "Error reading node properties", t);
}
return snapshot;
}
/**
* Safely recycles node info on Android API < 30 to prevent Binder pool exhaustion.
*/
@SuppressWarnings("deprecation")
public static void safeRecycle(AccessibilityNodeInfo node) {
if (node == null) return;
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.R) {
try {
node.recycle();
} catch (Throwable ignored) {}
}
}
/**
* Finds the currently focused or editable input node for text entry.
*/
public static AccessibilityNodeInfo findInputNode(AccessibilityService service) {
for (RawRootEntry entry : getActiveRawRoots(service)) {
try {
AccessibilityNodeInfo focused = entry.root.findFocus(AccessibilityNodeInfo.FOCUS_INPUT);
if (focused != null) return focused;
} catch (Throwable ignored) {}
}
for (RawRootEntry entry : getActiveRawRoots(service)) {
AccessibilityNodeInfo editable = findFirstEditable(entry.root);
if (editable != null) return editable;
}
return null;
}
private static AccessibilityNodeInfo findFirstEditable(AccessibilityNodeInfo node) {
if (node == null) return null;
try {
if (node.isEditable() && node.isFocusable() && node.isEnabled()) {
return node;
}
int count = node.getChildCount();
for (int i = 0; i < count; i++) {
AccessibilityNodeInfo child = node.getChild(i);
if (child != null) {
AccessibilityNodeInfo res = findFirstEditable(child);
if (res != null) return res;
safeRecycle(child);
}
}
} catch (Throwable ignored) {}
return null;
}
}
@@ -0,0 +1,102 @@
package com.artemis.helper;
/**
* High-performance XML serialization and character sanitization utilities.
* Ensures the output strictly conforms to the W3C XML 1.0 specification,
* preventing XML parsing errors (such as ET.ParseError in Python) caused by
* unprintable control characters, NUL bytes, or unescaped entities.
*/
public final class XmlUtils {
private XmlUtils() {}
/**
* Appends an escaped attribute to the StringBuilder: {@code name="value"}.
* If the value is null or empty, appends {@code name=""}.
*/
public static void appendAttribute(StringBuilder sb, String name, CharSequence value) {
sb.append(' ').append(name).append("=\"");
if (value != null && value.length() > 0) {
escapeXmlAttr(sb, value);
}
sb.append('"');
}
/**
* Appends a boolean attribute: {@code name="true"} or {@code name="false"}.
*/
public static void appendBooleanAttribute(StringBuilder sb, String name, boolean value) {
sb.append(' ').append(name).append(value ? "=\"true\"" : "=\"false\"");
}
/**
* Appends an integer attribute: {@code name="123"}.
*/
public static void appendIntAttribute(StringBuilder sb, String name, int value) {
sb.append(' ').append(name).append("=\"").append(value).append('"');
}
/**
* Escapes and sanitizes an attribute value according to XML 1.0 rules.
* Drops illegal XML characters (e.g. ASCII control characters 0x00-0x08, 0x0B, 0x0C, 0x0E-0x1F)
* and escapes standard entities (&amp;, &lt;, &gt;, &quot;, &apos;).
*/
public static void escapeXmlAttr(StringBuilder sb, CharSequence text) {
final int len = text.length();
for (int i = 0; i < len; i++) {
char c = text.charAt(i);
switch (c) {
case '&':
sb.append("&amp;");
break;
case '<':
sb.append("&lt;");
break;
case '>':
sb.append("&gt;");
break;
case '"':
sb.append("&quot;");
break;
case '\'':
sb.append("&apos;");
break;
case '\t':
sb.append("&#9;");
break;
case '\n':
sb.append("&#10;");
break;
case '\r':
sb.append("&#13;");
break;
default:
// XML 1.0 legal character range check:
// #x9 | #xA | #xD | [#x20-#xD7FF] | [#xE000-#xFFFD] | [#x10000-#x10FFFF]
if (c >= 0x20 && c <= 0xD7FF) {
sb.append(c);
} else if (c >= 0xE000 && c <= 0xFFFD) {
sb.append(c);
} else if (Character.isHighSurrogate(c)) {
if (i + 1 < len && Character.isLowSurrogate(text.charAt(i + 1))) {
sb.append(c).append(text.charAt(i + 1));
i++;
}
}
// Characters outside these ranges (e.g. \u0000 through \u0008, \u000B, \u000C, \u000E-\u001F,
// unpaired surrogates, \uFFFE, \uFFFF) are invalid XML 1.0 and silently omitted to guarantee well-formedness.
break;
}
}
}
/**
* Sanitizes a string for safe XML attribute inclusion, returning the escaped string.
*/
public static String escapeXmlAttr(CharSequence text) {
if (text == null || text.length() == 0) return "";
StringBuilder sb = new StringBuilder(text.length() + 16);
escapeXmlAttr(sb, text);
return sb.toString();
}
}
@@ -0,0 +1,4 @@
<resources>
<string name="app_name">Artemis Accessibility Helper</string>
<string name="accessibility_service_description">Lightweight non-exclusive accessibility helper for Artemis UI automation and testing.</string>
</resources>
@@ -0,0 +1,10 @@
<?xml version="1.0" encoding="utf-8"?>
<accessibility-service xmlns:android="http://schemas.android.com/apk/res/android"
android:accessibilityEventTypes="typeAllMask"
android:accessibilityFeedbackType="feedbackGeneric"
android:notificationTimeout="50"
android:accessibilityFlags="flagDefault|flagRetrieveInteractiveWindows|flagIncludeNotImportantViews|flagReportViewIds"
android:canRetrieveWindowContent="true"
android:canPerformGestures="true"
android:canTakeScreenshot="true"
android:description="@string/accessibility_service_description" />
@@ -0,0 +1,3 @@
plugins {
id("com.android.application") version "8.5.0" apply false
}
+118
View File
@@ -0,0 +1,118 @@
#!/usr/bin/env bash
set -e
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
cd "$DIR"
# 1. Locate Android SDK
SDK_ROOT="${ANDROID_HOME:-${ANDROID_SDK_ROOT:-$HOME/Library/Android/sdk}}"
if [ ! -d "$SDK_ROOT" ]; then
for candidate in /opt/android-sdk /usr/local/share/android-sdk "$HOME/Android/Sdk"; do
if [ -d "$candidate" ]; then
SDK_ROOT="$candidate"
break
fi
done
fi
if [ ! -d "$SDK_ROOT" ]; then
echo "Error: Android SDK not found at $SDK_ROOT. Set ANDROID_HOME." >&2
exit 1
fi
BUILD_TOOLS="$(find "$SDK_ROOT/build-tools" -maxdepth 1 -mindepth 1 | sort -V | tail -n 1)"
PLATFORM="$(find "$SDK_ROOT/platforms" -maxdepth 1 -mindepth 1 | sort -V | tail -n 1)"
if [ -z "$BUILD_TOOLS" ] || [ ! -d "$BUILD_TOOLS" ]; then
echo "Error: No build-tools found under $SDK_ROOT/build-tools" >&2
exit 1
fi
if [ -z "$PLATFORM" ] || [ ! -f "$PLATFORM/android.jar" ]; then
echo "Error: android.jar not found under $SDK_ROOT/platforms" >&2
exit 1
fi
# 2. Locate Java compiler and runtime
if [ -z "$JAVA_HOME" ]; then
if [ -d "/Applications/Android Studio.app/Contents/jbr/Contents/Home" ]; then
export JAVA_HOME="/Applications/Android Studio.app/Contents/jbr/Contents/Home"
fi
fi
if [ -n "$JAVA_HOME" ]; then
export PATH="$JAVA_HOME/bin:$PATH"
fi
if ! command -v javac >/dev/null 2>&1; then
echo "Error: javac not found in PATH or JAVA_HOME" >&2
exit 1
fi
echo "=================================================="
echo " Building ArtemisAccessibilityHelper.apk"
echo " Build Tools: $(basename "$BUILD_TOOLS")"
echo " Platform: $(basename "$PLATFORM")"
echo " Java: $(javac -version 2>&1)"
echo "=================================================="
rm -rf build
mkdir -p build/gen build/obj build/apk build/res_compiled
echo "-> Compiling resources..."
"$BUILD_TOOLS/aapt2" compile --dir app/src/main/res -o build/res_compiled/res.zip
echo "-> Linking APK..."
"$BUILD_TOOLS/aapt2" link \
-I "$PLATFORM/android.jar" \
--manifest app/src/main/AndroidManifest.xml \
--min-sdk-version 24 \
--target-sdk-version 35 \
--java build/gen \
-o build/apk/unaligned.apk \
build/res_compiled/res.zip
echo "-> Compiling Java sources..."
JAVA_FILES="$(find build/gen app/src/main/java -name "*.java")"
javac -encoding UTF-8 \
-cp "$PLATFORM/android.jar" \
-d build/obj \
$JAVA_FILES
echo "-> Dexing with D8..."
CLASS_FILES="$(find build/obj -name "*.class")"
"$BUILD_TOOLS/d8" \
--lib "$PLATFORM/android.jar" \
--output build/apk/ \
--min-api 24 \
$CLASS_FILES
echo "-> Packaging DEX..."
cd build/apk
zip -u -q unaligned.apk classes.dex
cd "$DIR"
echo "-> Zipaligning..."
"$BUILD_TOOLS/zipalign" -f -p 4 build/apk/unaligned.apk build/apk/aligned.apk
echo "-> Signing APK..."
if [ ! -f debug.keystore ]; then
echo "Creating persistent debug.keystore..."
keytool -genkey -v -keystore debug.keystore \
-storepass android -alias androiddebugkey -keypass android \
-keyalg RSA -keysize 2048 -validity 10000 \
-dname "CN=Artemis Debug,O=Artemis,C=US"
fi
"$BUILD_TOOLS/apksigner" sign \
--ks debug.keystore \
--ks-pass pass:android \
--key-pass pass:android \
--ks-key-alias androiddebugkey \
--out ArtemisAccessibilityHelper.apk \
build/apk/aligned.apk
echo "=================================================="
echo " Build SUCCESSFUL!"
echo " Artifact: $DIR/ArtemisAccessibilityHelper.apk"
ls -lh ArtemisAccessibilityHelper.apk
echo "=================================================="
Binary file not shown.
@@ -0,0 +1,2 @@
org.gradle.jvmargs=-Xmx2048m -Dfile.encoding=UTF-8
android.useAndroidX=true
@@ -0,0 +1,17 @@
pluginManagement {
repositories {
google()
mavenCentral()
gradlePluginPortal()
}
}
dependencyResolutionManagement {
repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS)
repositories {
google()
mavenCentral()
}
}
rootProject.name = "artemis-accessibility-helper"
include(":app")
@@ -0,0 +1,88 @@
import unittest
from unittest.mock import MagicMock, patch
from artemis.clients.accessibility_client import AccessibilityClient, DEFAULT_PORT
class TestAccessibilityClient(unittest.TestCase):
def setUp(self):
self.client = AccessibilityClient(device_id="dummy_device", local_port=DEFAULT_PORT)
@patch("artemis.clients.accessibility_client.urllib.request.urlopen")
def test_ping_success(self, mock_urlopen):
mock_resp = MagicMock()
mock_resp.read.return_value = b'{"success": true, "service": "ArtemisAccessibilityService"}'
mock_resp.__enter__.return_value = mock_resp
mock_urlopen.return_value = mock_resp
self.assertTrue(self.client.ping())
@patch("artemis.clients.accessibility_client.urllib.request.urlopen")
def test_get_hierarchy(self, mock_urlopen):
mock_resp = MagicMock()
mock_resp.read.return_value = b'{"success": true, "elements": [{"text": "OK"}]}'
mock_resp.__enter__.return_value = mock_resp
mock_urlopen.return_value = mock_resp
data = self.client.get_hierarchy()
self.assertTrue(data["success"])
self.assertEqual(data["elements"][0]["text"], "OK")
@patch("artemis.clients.accessibility_client.urllib.request.urlopen")
def test_tap_action(self, mock_urlopen):
mock_resp = MagicMock()
mock_resp.read.return_value = b'{"success": true}'
mock_resp.__enter__.return_value = mock_resp
mock_urlopen.return_value = mock_resp
@patch("artemis.clients.accessibility_client.urllib.request.urlopen")
def test_get_hierarchy_xml(self, mock_urlopen):
mock_resp = MagicMock()
mock_resp.read.return_value = b"<hierarchy rotation=\"0\"><node text=\"Save\" /></hierarchy>"
mock_resp.__enter__.return_value = mock_resp
mock_urlopen.return_value = mock_resp
xml = self.client.get_hierarchy_xml()
self.assertIn("<hierarchy", xml)
self.assertIn("Save", xml)
@patch.object(AccessibilityClient, "get_atomic_snapshot")
def test_get_screen_data_atomic(self, mock_atomic):
mock_atomic.return_value = {
"success": True,
"has_screenshot": True,
"screenshot_base64": "dummy_b64",
"xml": "<hierarchy rotation=\"0\"><node text=\"OK\" /></hierarchy>",
"elements": [{"text": "OK"}],
"width": 1080,
"height": 2400,
}
screen_data = self.client.get_screen_data()
self.assertEqual(screen_data.width, 1080)
self.assertEqual(screen_data.height, 2400)
self.assertEqual(screen_data.base64, "dummy_b64")
self.assertIn("rotation=\"0\"", screen_data.hierarchy_xml)
self.assertEqual(len(screen_data.elements), 1)
@patch.object(AccessibilityClient, "get_atomic_snapshot", return_value=None)
@patch.object(AccessibilityClient, "get_screenshot")
@patch.object(AccessibilityClient, "get_hierarchy")
def test_get_screen_data_fallback(self, mock_get_hierarchy, mock_get_screenshot, _mock_atomic):
from PIL import Image
mock_get_screenshot.return_value = Image.new("RGB", (1080, 2400))
mock_get_hierarchy.return_value = {
"success": True,
"xml": "<hierarchy rotation=\"0\"><node text=\"Fallback\" /></hierarchy>",
"elements": [{"text": "Fallback"}],
}
screen_data = self.client.get_screen_data()
self.assertEqual(screen_data.width, 1080)
self.assertEqual(screen_data.height, 2400)
self.assertIn("Fallback", screen_data.hierarchy_xml)
self.assertEqual(len(screen_data.elements), 1)
if __name__ == "__main__":
unittest.main()