chenmingxuan/Artemis-Eapil
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 32 Wiki Activity

feat: add managed accessibility helper with UIAutomator fallback

Browse Source
This commit is contained in:
somew1nd committed 2026-09-10 19:40:26 -07:00
1 parent b86c37127d
commit 52749c7f23
53 files changed
+5935 -536
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

packages/artemis-accessibility-helper/ArtemisAccessibilityHelper.apk
BIN
View File
Binary file not shown.
packages/artemis-accessibility-helper/app/build.gradle.kts
+2 -2
View File
@@ -10,8 +10,8 @@ android {
applicationId = "com.artemis.helper"
minSdk = 24
targetSdk = 35
versionCode = 1
versionName = "1.0.0"
versionCode = 6
versionName = "1.2.0"
}
buildTypes {
packages/artemis-accessibility-helper/app/src/main/AndroidManifest.xml
+12
View File
@@ -32,6 +32,18 @@
android:value="Artemis mobile automation test bridge" />
</service>
<!-- Session token drop: only a sender holding WRITE_SECURE_SETTINGS (the adb
shell user, the system) can deliver this broadcast, so no ordinary app on
the device can set a token of its own and then talk to the loopback server. -->
<receiver
android:name=".TokenReceiver"
android:exported="true"
android:permission="android.permission.WRITE_SECURE_SETTINGS">
<intent-filter>
<action android:name="com.artemis.helper.SET_TOKEN" />
</intent-filter>
</receiver>
</application>
</manifest>
packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/A11yNode.java
+12 -2
View File
@@ -1,6 +1,5 @@
package com.artemis.helper;
import android.graphics.Rect;
import org.json.JSONArray;
import org.json.JSONObject;
@@ -12,6 +11,12 @@ import java.util.List;
* Decoupled from live Android AccessibilityNodeInfo objects to prevent
* Binder proxy memory leaks, avoid stale node crashes, and ensure
* instant thread-safe serialization to both XML and JSON.
*
* {@code bounds} are the node's <em>visible</em> bounds: the raw
* {@code getBoundsInScreen} rectangle intersected with the display, the
* node's window and every scrollable ancestor, exactly like UIAutomator's
* {@code getVisibleBoundsInScreen}. They are therefore never negative and
* never extend past the screen.
*/
public final class A11yNode {
@@ -32,6 +37,7 @@ public final class A11yNode {
public boolean longClickable = false;
public boolean password = false;
public boolean selected = false;
public boolean visibleToUser = true;
// Window metadata for multi-window awareness
public int windowId = -1;
@@ -94,9 +100,12 @@ public final class A11yNode {
XmlUtils.appendBooleanAttribute(sb, "long-clickable", longClickable);
XmlUtils.appendBooleanAttribute(sb, "password", password);
XmlUtils.appendBooleanAttribute(sb, "selected", selected);
XmlUtils.appendBooleanAttribute(sb, "visible-to-user", visibleToUser);
XmlUtils.appendAttribute(sb, "bounds", getBoundsString());
XmlUtils.appendIntAttribute(sb, "drawing-order", drawingOrder);
XmlUtils.appendAttribute(sb, "hint", hint);
if (!hint.isEmpty()) {
XmlUtils.appendAttribute(sb, "hint", hint);
}
// Window metadata (emitted on window root nodes)
if (windowId >= 0) {
@@ -199,6 +208,7 @@ public final class A11yNode {
obj.put("selected", selected);
obj.put("password", password);
obj.put("long-clickable", longClickable);
obj.put("visible-to-user", visibleToUser);
obj.put("drawing-order", drawingOrder);
if (!hint.isEmpty()) {
obj.put("hint", hint);
packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/ArtemisAccessibilityService.java
+64 -9
View File
@@ -7,6 +7,7 @@ import android.app.NotificationChannel;
import android.app.NotificationManager;
import android.content.Context;
import android.content.Intent;
import android.content.pm.PackageInfo;
import android.content.pm.ServiceInfo;
import android.os.Build;
import android.util.Log;
@@ -17,12 +18,24 @@ import android.view.accessibility.AccessibilityEvent;
*
* Runs concurrently with any Android UI testing framework (Mobly, Appium, Espresso)
* without monopolizing Android's singleton UiAutomationService connection.
*
* The service subscribes to window-state changes only: that is the single event it
* uses (to remember the foreground package / activity), and anything wider costs
* CPU and battery on a device that is idle between tasks.
*/
public class ArtemisAccessibilityService extends AccessibilityService {
private static final String TAG = "ArtemisA11yService";
public static final int DEFAULT_PORT = 18888;
private static final String CHANNEL_ID = "artemis_helper_channel";
/**
* Bumped whenever the HTTP contract changes in a way an older host cannot use.
* 2: token authentication, byte-accurate bodies, visible-only dumps, fields=.
*/
public static final int PROTOCOL_VERSION = 2;
// Channel importance is frozen by Android once a channel exists, so a quieter
// channel needs a new id; the pre-1.1.3 channel is deleted on start.
private static final String LEGACY_CHANNEL_ID = "artemis_helper_channel";
private static final String CHANNEL_ID = "artemis_helper_quiet";
private static final int NOTIFICATION_ID = 18888;
private static volatile ArtemisAccessibilityService instance;
@@ -43,6 +56,29 @@ public class ArtemisAccessibilityService extends AccessibilityService {
return currentActivityName;
}
/** Installed helper versionCode, or -1 when the package manager cannot answer. */
@SuppressWarnings("deprecation")
public long getVersionCode() {
try {
PackageInfo info = getPackageManager().getPackageInfo(getPackageName(), 0);
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) {
return info.getLongVersionCode();
}
return info.versionCode;
} catch (Throwable t) {
return -1L;
}
}
public String getVersionName() {
try {
PackageInfo info = getPackageManager().getPackageInfo(getPackageName(), 0);
return info.versionName == null ? "" : info.versionName;
} catch (Throwable t) {
return "";
}
}
@Override
public void onServiceConnected() {
super.onServiceConnected();
@@ -58,12 +94,14 @@ public class ArtemisAccessibilityService extends AccessibilityService {
if (info == null) {
info = new AccessibilityServiceInfo();
}
info.eventTypes = AccessibilityEvent.TYPES_ALL_MASK;
info.eventTypes = AccessibilityEvent.TYPE_WINDOW_STATE_CHANGED;
info.feedbackType = AccessibilityServiceInfo.FEEDBACK_GENERIC;
info.notificationTimeout = 50;
info.notificationTimeout = 100;
info.flags |= AccessibilityServiceInfo.FLAG_RETRIEVE_INTERACTIVE_WINDOWS
| AccessibilityServiceInfo.FLAG_INCLUDE_NOT_IMPORTANT_VIEWS
| AccessibilityServiceInfo.FLAG_REPORT_VIEW_IDS;
// Not-important views are what UIAutomator's compressed dump leaves out;
// keep them out here too so both backends describe the same tree.
info.flags &= ~AccessibilityServiceInfo.FLAG_INCLUDE_NOT_IMPORTANT_VIEWS;
setServiceInfo(info);
Log.i(TAG, "AccessibilityServiceInfo flags dynamically enforced");
} catch (Throwable t) {
@@ -94,12 +132,19 @@ public class ArtemisAccessibilityService extends AccessibilityService {
if (nm == null) return;
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
try {
nm.deleteNotificationChannel(LEGACY_CHANNEL_ID);
} catch (Throwable ignored) {}
// Ask for IMPORTANCE_MIN (no sound, collapsed in the shade). Android raises
// foreground-service channels to LOW at most, which is still silent. The
// notification exists only to keep the service alive on aggressive ROMs;
// it must not look like something the person needs to act on.
NotificationChannel channel = new NotificationChannel(
CHANNEL_ID,
"Artemis Helper Service",
NotificationManager.IMPORTANCE_LOW
"Artemis test helper",
NotificationManager.IMPORTANCE_MIN
);
channel.setDescription("Keeps Artemis Accessibility Helper active for automation testing");
channel.setDescription("Shown while the Artemis test helper is installed on this device");
channel.setShowBadge(false);
nm.createNotificationChannel(channel);
}
@@ -111,10 +156,20 @@ public class ArtemisAccessibilityService extends AccessibilityService {
builder = new Notification.Builder(this);
}
builder.setContentTitle("Artemis Accessibility Helper")
.setContentText("Active on 127.0.0.1:" + DEFAULT_PORT)
builder.setContentTitle("Artemis test helper is running")
.setContentText("Lets Artemis read this screen during automated tests. "
+ "Remove with: artemis helper uninstall")
.setStyle(new Notification.BigTextStyle().bigText(
"Installed by Artemis to read the screen layout during automated "
+ "tests. It answers only on this device (127.0.0.1:" + DEFAULT_PORT
+ "), only to the computer that is connected over USB debugging and "
+ "holds the session token, and it sends nothing anywhere. Remove it "
+ "any time with `artemis helper uninstall` or from Settings > Apps."))
.setSmallIcon(android.R.drawable.stat_notify_sync)
.setOngoing(true);
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) {
builder.setPriority(Notification.PRIORITY_MIN);
}
if (Build.VERSION.SDK_INT >= 34) {
startForeground(NOTIFICATION_ID, builder.build(), ServiceInfo.FOREGROUND_SERVICE_TYPE_SPECIAL_USE);
packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/CommandServer.java
+241 -80
View File
@@ -3,27 +3,46 @@ package com.artemis.helper;
import android.util.Log;
import org.json.JSONObject;
import java.io.BufferedReader;
import java.io.BufferedInputStream;
import java.io.ByteArrayOutputStream;
import java.io.Closeable;
import java.io.InputStreamReader;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.net.Socket;
import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
/**
* High-performance, multi-threaded command server bound strictly to loopback (127.0.0.1:18888).
* Supports:
* - HTTP REST API (/dump, /dump_xml, /hierarchy.xml, /ping, /action)
* - Line-delimited JSON-RPC over TCP socket
* Multi-threaded command server bound strictly to loopback (127.0.0.1:18888).
*
* Protocols:
* - HTTP (/ping, /dump, /dump_xml, /hierarchy.xml, /snapshot, /action)
* - Line-delimited JSON-RPC over a raw TCP socket
*
* Every endpoint except {@code /ping} requires the session token the host pushed
* through {@link TokenReceiver} (header {@code X-Artemis-Token}, query parameter
* {@code token}, or JSON field {@code token}). Loopback is reachable by every app
* on the device, so without the token this service would hand any local app a
* full-screen reader and a gesture injector.
*
* Request bodies are read as bytes (Content-Length is a byte count), so UTF-8
* payloads of any script are handled exactly.
*/
public class CommandServer extends Thread {
private static final String TAG = "ArtemisCommandServer";
private static final int MAX_HEADER_LINE = 16 * 1024;
private static final int MAX_BODY = 4 * 1024 * 1024;
private static final String TOKEN_HEADER = "x-artemis-token";
private final ArtemisAccessibilityService service;
private final int port;
private volatile boolean isRunning = true;
@@ -67,25 +86,66 @@ public class CommandServer extends Thread {
}
}
// ------------------------------------------------------------------ //
// Byte-accurate request reading
// ------------------------------------------------------------------ //
/** Reads one line (without CR/LF) as raw bytes; null at end of stream. */
private static byte[] readLineBytes(InputStream in) throws Exception {
ByteArrayOutputStream line = new ByteArrayOutputStream(256);
int b;
while ((b = in.read()) >= 0) {
if (b == '\n') {
break;
}
if (b != '\r') {
line.write(b);
}
if (line.size() > MAX_HEADER_LINE) {
throw new IllegalStateException("Header line too long");
}
}
if (b < 0 && line.size() == 0) {
return null;
}
return line.toByteArray();
}
private static byte[] readExactly(InputStream in, int length) throws Exception {
byte[] buf = new byte[length];
int total = 0;
while (total < length) {
int r = in.read(buf, total, length - total);
if (r < 0) break;
total += r;
}
if (total < length) {
byte[] shorter = new byte[total];
System.arraycopy(buf, 0, shorter, 0, total);
return shorter;
}
return buf;
}
private void handleClient(Socket socket) {
try {
socket.setSoTimeout(10000);
BufferedReader reader = new BufferedReader(new InputStreamReader(socket.getInputStream(), StandardCharsets.UTF_8));
InputStream in = new BufferedInputStream(socket.getInputStream());
OutputStream output = socket.getOutputStream();
String firstLine = reader.readLine();
byte[] firstLine = readLineBytes(in);
if (firstLine == null) return;
String trimmed = firstLine.trim();
String ascii = new String(firstLine, StandardCharsets.ISO_8859_1).trim();
if (trimmed.startsWith("GET ") || trimmed.startsWith("POST ")) {
handleHttp(trimmed, reader, output);
} else if (trimmed.startsWith("{")) {
handleJsonRpc(trimmed, output);
if (ascii.startsWith("GET ") || ascii.startsWith("POST ")) {
handleHttp(ascii, in, output);
} else if (ascii.startsWith("{")) {
handleJsonRpc(new String(firstLine, StandardCharsets.UTF_8).trim(), output);
} else {
JSONObject err = new JSONObject();
err.put("success", false);
err.put("error", "Unsupported protocol");
sendJsonResponse(output, err.toString());
sendJson(output, 400, err.toString());
}
} catch (Exception e) {
Log.w(TAG, "Client handling error: " + e.getMessage());
@@ -94,82 +154,131 @@ public class CommandServer extends Thread {
}
}
private void handleHttp(String requestLine, BufferedReader reader, OutputStream output) {
// ------------------------------------------------------------------ //
// HTTP
// ------------------------------------------------------------------ //
private static Map<String, String> parseQuery(String query) {
Map<String, String> params = new HashMap<>();
if (query == null || query.isEmpty()) return params;
for (String pair : query.split("&")) {
if (pair.isEmpty()) continue;
int eq = pair.indexOf('=');
try {
String key = URLDecoder.decode(eq < 0 ? pair : pair.substring(0, eq), "UTF-8");
String value = eq < 0 ? "" : URLDecoder.decode(pair.substring(eq + 1), "UTF-8");
params.put(key, value);
} catch (Exception ignored) {}
}
return params;
}
private void handleHttp(String requestLine, InputStream in, OutputStream output) {
try {
String[] parts = requestLine.split(" ");
String path = parts.length > 1 ? parts[1] : "/";
String target = parts.length > 1 ? parts[1] : "/";
String path = target;
Map<String, String> query;
int q = target.indexOf('?');
if (q >= 0) {
path = target.substring(0, q);
query = parseQuery(target.substring(q + 1));
} else {
query = new HashMap<>();
}
int contentLength = 0;
String line;
while ((line = reader.readLine()) != null) {
if (line.isEmpty()) break;
String lower = line.toLowerCase();
if (lower.startsWith("content-length:")) {
String headerToken = null;
byte[] lineBytes;
while ((lineBytes = readLineBytes(in)) != null) {
if (lineBytes.length == 0) break;
String line = new String(lineBytes, StandardCharsets.ISO_8859_1);
int colon = line.indexOf(':');
if (colon < 0) continue;
String name = line.substring(0, colon).trim().toLowerCase(Locale.ROOT);
String value = line.substring(colon + 1).trim();
if (name.equals("content-length")) {
try {
contentLength = Integer.parseInt(lower.substring("content-length:".length()).trim());
contentLength = Integer.parseInt(value);
} catch (Exception ignored) {}
} else if (name.equals(TOKEN_HEADER)) {
headerToken = value;
}
}
if (contentLength < 0 || contentLength > MAX_BODY) {
sendJson(output, 413, errorJson("Body too large").toString());
return;
}
String body = "";
if (contentLength > 0) {
char[] buf = new char[contentLength];
int total = 0;
while (total < contentLength) {
int r = reader.read(buf, total, contentLength - total);
if (r < 0) break;
total += r;
}
body = new String(buf, 0, total);
body = new String(readExactly(in, contentLength), StandardCharsets.UTF_8);
}
String token = headerToken != null ? headerToken : query.get("token");
boolean authed = TokenStore.matches(token);
if (path.equals("/ping") || path.equals("/")) {
sendJson(output, 200, buildPing(authed).toString());
return;
}
if (!authed) {
sendJson(output, 401, unauthorizedJson().toString());
return;
}
// Route endpoints
if (path.startsWith("/snapshot")) {
JSONObject json = HierarchyDumper.dumpAtomicSnapshot(service);
sendJsonResponse(output, json.toString());
} else if (path.equals("/dump_xml") || path.equals("/hierarchy.xml") || path.startsWith("/dump?format=xml")) {
String xml = HierarchyDumper.dumpXml(service);
sendXmlResponse(output, xml);
} else if (path.startsWith("/dump") || path.startsWith("/hierarchy")) {
JSONObject json = HierarchyDumper.dump(service);
sendJsonResponse(output, json.toString());
} else if (path.startsWith("/ping") || path.equals("/")) {
JSONObject r = new JSONObject();
r.put("success", true);
r.put("service", "ArtemisAccessibilityService");
r.put("package", service.getCurrentPackageName());
r.put("activity", service.getCurrentActivityName());
sendJsonResponse(output, r.toString());
} else if (path.startsWith("/action") || path.startsWith("/rpc")) {
if (path.equals("/snapshot")) {
HierarchyDumper.DumpOptions options = HierarchyDumper.DumpOptions.forSnapshot().apply(query);
JSONObject json = HierarchyDumper.dumpAtomicSnapshot(service, options);
sendJson(output, 200, json.toString());
} else if (path.equals("/dump_xml") || path.equals("/hierarchy.xml")
|| (path.equals("/dump") && "xml".equals(query.get("format")))) {
HierarchyDumper.DumpOptions options = HierarchyDumper.DumpOptions.forSnapshot().apply(query);
sendXml(output, HierarchyDumper.dumpXml(service, options));
} else if (path.equals("/dump") || path.equals("/hierarchy")) {
HierarchyDumper.DumpOptions options = HierarchyDumper.DumpOptions.forDump().apply(query);
sendJson(output, 200, HierarchyDumper.dump(service, options).toString());
} else if (path.equals("/action") || path.equals("/rpc")) {
JSONObject json = body.isEmpty() ? new JSONObject() : new JSONObject(body);
JSONObject resp = executeCommand(json.optString("cmd", ""), json);
sendJsonResponse(output, resp.toString());
sendJson(output, 200, resp.toString());
} else {
JSONObject r = new JSONObject();
r.put("success", false);
r.put("error", "Unknown endpoint: " + path);
sendJsonResponse(output, r.toString());
sendJson(output, 404, errorJson("Unknown endpoint: " + path).toString());
}
} catch (Exception e) {
Log.e(TAG, "HTTP error", e);
try {
sendJson(output, 500, errorJson("Internal error: " + e.getMessage()).toString());
} catch (Throwable ignored) {}
}
}
// ------------------------------------------------------------------ //
// Raw JSON-RPC
// ------------------------------------------------------------------ //
private void handleJsonRpc(String line, OutputStream output) {
try {
JSONObject json = new JSONObject(line);
String cmd = json.optString("cmd", "");
JSONObject resp = executeCommand(cmd, json);
boolean authed = TokenStore.matches(json.optString("token", null));
JSONObject resp;
if (cmd.equalsIgnoreCase("ping")) {
resp = buildPing(authed);
} else if (!authed) {
resp = unauthorizedJson();
} else {
resp = executeCommand(cmd, json);
}
byte[] bytes = resp.toString().getBytes(StandardCharsets.UTF_8);
output.write(bytes);
output.write('\n');
output.flush();
} catch (Exception e) {
try {
JSONObject err = new JSONObject();
err.put("success", false);
err.put("error", "JSON parse error: " + e.getMessage());
JSONObject err = errorJson("JSON parse error: " + e.getMessage());
output.write(err.toString().getBytes(StandardCharsets.UTF_8));
output.write('\n');
output.flush();
@@ -177,21 +286,21 @@ public class CommandServer extends Thread {
}
}
// ------------------------------------------------------------------ //
// Commands
// ------------------------------------------------------------------ //
private JSONObject executeCommand(String cmd, JSONObject params) {
JSONObject resp = new JSONObject();
try {
switch (cmd.toLowerCase()) {
switch (cmd.toLowerCase(Locale.ROOT)) {
case "ping":
resp.put("success", true);
resp.put("service", "ArtemisAccessibilityService");
resp.put("package", service.getCurrentPackageName());
resp.put("activity", service.getCurrentActivityName());
break;
return buildPing(true);
case "dump":
case "dump_ui":
return HierarchyDumper.dump(service);
return HierarchyDumper.dump(service, HierarchyDumper.DumpOptions.forDump());
case "snapshot":
return HierarchyDumper.dumpAtomicSnapshot(service);
return HierarchyDumper.dumpAtomicSnapshot(service, HierarchyDumper.DumpOptions.forSnapshot());
case "dump_xml":
resp.put("success", true);
resp.put("xml", HierarchyDumper.dumpXml(service));
@@ -239,11 +348,15 @@ public class CommandServer extends Thread {
break;
case "type":
String text = params.optString("text", "");
resp.put("success", GestureController.setText(service, text));
boolean append = params.optBoolean("append", false);
resp.put("success", GestureController.setText(service, text, append));
break;
case "clear":
resp.put("success", GestureController.clearText(service));
break;
case "clipboard":
resp.put("success", GestureController.setClipboard(service, params.optString("text", "")));
break;
case "global":
String action = params.optString("action", "");
resp.put("success", GestureController.performGlobalAction(service, action));
@@ -262,11 +375,64 @@ public class CommandServer extends Thread {
return resp;
}
private void sendJsonResponse(OutputStream output, String jsonStr) {
/**
* Health payload shared by GET /ping and the "ping" RPC. The host compares
* version_code / protocol_version against the bundled APK to decide whether
* to upgrade, and token_set to decide whether to push its token. The
* foreground package and activity are only disclosed to an authenticated caller.
*/
private JSONObject buildPing(boolean authed) throws Exception {
JSONObject r = new JSONObject();
r.put("success", true);
r.put("service", "ArtemisAccessibilityService");
r.put("version_code", service.getVersionCode());
r.put("version_name", service.getVersionName());
r.put("protocol_version", ArtemisAccessibilityService.PROTOCOL_VERSION);
r.put("port", port);
r.put("auth_required", true);
r.put("token_set", TokenStore.isSet());
r.put("authenticated", authed);
if (authed) {
r.put("package", service.getCurrentPackageName());
r.put("activity", service.getCurrentActivityName());
}
return r;
}
private static JSONObject errorJson(String message) throws Exception {
JSONObject r = new JSONObject();
r.put("success", false);
r.put("error", message);
return r;
}
private static JSONObject unauthorizedJson() throws Exception {
JSONObject r = errorJson(TokenStore.isSet()
? "unauthorized: wrong or missing X-Artemis-Token"
: "unauthorized: no session token has been pushed to the helper yet");
r.put("token_set", TokenStore.isSet());
return r;
}
// ------------------------------------------------------------------ //
// Responses
// ------------------------------------------------------------------ //
private static String reason(int status) {
switch (status) {
case 200: return "OK";
case 400: return "Bad Request";
case 401: return "Unauthorized";
case 404: return "Not Found";
case 413: return "Payload Too Large";
default: return "Internal Server Error";
}
}
private static void send(OutputStream output, int status, String contentType, byte[] bytes) {
try {
byte[] bytes = jsonStr.getBytes(StandardCharsets.UTF_8);
String header = "HTTP/1.1 200 OK\r\n" +
"Content-Type: application/json; charset=utf-8\r\n" +
String header = "HTTP/1.1 " + status + " " + reason(status) + "\r\n" +
"Content-Type: " + contentType + "; charset=utf-8\r\n" +
"Content-Length: " + bytes.length + "\r\n" +
"Connection: close\r\n\r\n";
output.write(header.getBytes(StandardCharsets.UTF_8));
@@ -275,17 +441,12 @@ public class CommandServer extends Thread {
} catch (Exception ignored) {}
}
private void sendXmlResponse(OutputStream output, String xmlStr) {
try {
byte[] bytes = xmlStr.getBytes(StandardCharsets.UTF_8);
String header = "HTTP/1.1 200 OK\r\n" +
"Content-Type: application/xml; charset=utf-8\r\n" +
"Content-Length: " + bytes.length + "\r\n" +
"Connection: close\r\n\r\n";
output.write(header.getBytes(StandardCharsets.UTF_8));
output.write(bytes);
output.flush();
} catch (Exception ignored) {}
private static void sendJson(OutputStream output, int status, String jsonStr) {
send(output, status, "application/json", jsonStr.getBytes(StandardCharsets.UTF_8));
}
private static void sendXml(OutputStream output, String xmlStr) {
send(output, 200, "application/xml", xmlStr.getBytes(StandardCharsets.UTF_8));
}
public void shutdown() {
packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/GestureController.java
+59 -1
View File
@@ -2,6 +2,9 @@ package com.artemis.helper;
import android.accessibilityservice.AccessibilityService;
import android.accessibilityservice.GestureDescription;
import android.content.ClipData;
import android.content.ClipboardManager;
import android.content.Context;
import android.graphics.Path;
import android.os.Build;
import android.os.Bundle;
@@ -65,14 +68,36 @@ public final class GestureController {
}
public static boolean setText(AccessibilityService service, String text) {
return setText(service, text, false);
}
/**
* Sets (or, with {@code append}, extends) the text of the focused / first editable field.
* ACTION_SET_TEXT replaces the whole content; appending mirrors what typing through an
* IME does, which is the contract the host's send_text has always had.
*/
public static boolean setText(AccessibilityService service, String text, boolean append) {
AccessibilityNodeInfo inputNode = HierarchyDumper.findInputNode(service);
if (inputNode == null) {
Log.w(TAG, "No editable/focused input node found for setText");
return false;
}
try {
String value = text == null ? "" : text;
if (append) {
CharSequence existing = inputNode.getText();
boolean showingHint = false;
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
try {
showingHint = inputNode.isShowingHintText();
} catch (Throwable ignored) {}
}
if (existing != null && existing.length() > 0 && !showingHint) {
value = existing.toString() + value;
}
}
Bundle args = new Bundle();
args.putCharSequence(AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, text);
args.putCharSequence(AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, value);
return inputNode.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, args);
} catch (Throwable t) {
Log.w(TAG, "performAction ACTION_SET_TEXT failed", t);
@@ -171,4 +196,37 @@ public final class GestureController {
return false;
}
}
/**
* Write text to the system clipboard. Clipboard writes are allowed from any
* app on every API level (only reads are restricted since Android 10), so this
* is the IME-free path the host uses for multiline and non-ASCII input:
* set the clip here, then KEYCODE_PASTE over adb.
*/
public static boolean setClipboard(final AccessibilityService service, final String text) {
final CountDownLatch latch = new CountDownLatch(1);
final AtomicBoolean ok = new AtomicBoolean(false);
MAIN_HANDLER.post(new Runnable() {
@Override
public void run() {
try {
ClipboardManager cm = (ClipboardManager) service.getSystemService(Context.CLIPBOARD_SERVICE);
if (cm != null) {
cm.setPrimaryClip(ClipData.newPlainText("artemis", text == null ? "" : text));
ok.set(true);
}
} catch (Throwable t) {
Log.w(TAG, "setClipboard failed: " + t.getMessage());
} finally {
latch.countDown();
}
}
});
try {
latch.await(2000, TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
return ok.get();
}
}
packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/HierarchyDumper.java
+315 -115
View File
@@ -21,11 +21,13 @@ import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
/**
@@ -34,73 +36,132 @@ import java.util.concurrent.atomic.AtomicReference;
* Capabilities:
* 1. Zero WaitForIdle hangs: Bypasses UIAutomator's waitForIdle() timeout.
* 2. Multi-window penetration: Captures App, Dialog, System UI, IME Keyboard, Split-Screen.
* 3. Rich semantic extraction: Captures errorText, isHeading, editable, paneTitle, tooltip.
* 4. Atomic screenshot snapshot: On API 30+, captures hardware bitmap and DOM tree simultaneously.
* 5. W3C XML 1.0 compliance: Strict character sanitization prevents Python parse failures.
* 3. UIAutomator parity: invisible nodes are skipped and bounds are clipped to the
* display, the window and scrollable ancestors, so the tree describes what is on
* screen and nothing else. {@code include_invisible=1} keeps everything (debugging).
* 4. Rich semantic extraction: Captures errorText, isHeading, editable, paneTitle, tooltip.
* 5. Atomic screenshot snapshot: On API 30+, captures hardware bitmap and DOM tree simultaneously.
* 6. W3C XML 1.0 compliance: Strict character sanitization prevents Python parse failures.
* 7. Batched IPC on API 33+: descendants are prefetched per getChild call instead of one
* Binder round trip per node.
*/
public final class HierarchyDumper {
private static final String TAG = "ArtemisHierarchyDumper";
private static final int MAX_DEPTH = 75;
private static final int MAX_NODES = 8000;
private static final int RETRY_COUNT = 3;
private static final long RETRY_INTERVAL_MS = 40L;
private static final ExecutorService SCREENSHOT_EXECUTOR = Executors.newSingleThreadExecutor();
/** Prefetch strategy for API 33+ ({@code AccessibilityNodeInfo.FLAG_PREFETCH_DESCENDANTS_HYBRID}). */
private static final int PREFETCH_DESCENDANTS_HYBRID = 1 << 3;
private HierarchyDumper() {}
/**
* Dumps the complete hierarchy as a JSON response.
* What a dump should contain. Every field defaults to the cheapest useful answer.
*/
public static final class DumpOptions {
public boolean includeInvisible = false;
public boolean wantXml = true;
public boolean wantElements = false;
public boolean wantTree = false;
public static DumpOptions forDump() {
DumpOptions o = new DumpOptions();
o.wantElements = true;
o.wantTree = true;
return o;
}
public static DumpOptions forSnapshot() {
return new DumpOptions();
}
/** Applies {@code fields=xml,elements,tree} and {@code include_invisible=1}. */
public DumpOptions apply(Map<String, String> query) {
if (query == null) return this;
String fields = query.get("fields");
if (fields != null && !fields.trim().isEmpty()) {
wantXml = wantElements = wantTree = false;
for (String f : fields.split(",")) {
String key = f.trim().toLowerCase();
if (key.equals("xml")) wantXml = true;
else if (key.equals("elements")) wantElements = true;
else if (key.equals("tree")) wantTree = true;
}
}
String inv = query.get("include_invisible");
if (inv != null) {
includeInvisible = inv.equals("1") || inv.equalsIgnoreCase("true");
}
return this;
}
}
/** Per-dump counters reported back to the host. */
private static final class DumpStats {
int nodes = 0;
int skippedInvisible = 0;
boolean truncated = false;
}
public static JSONObject dump(AccessibilityService service) {
return dump(service, DumpOptions.forDump());
}
/**
* Dumps the hierarchy as a JSON response containing the requested representations.
*/
public static JSONObject dump(AccessibilityService service, DumpOptions options) {
long startTime = System.currentTimeMillis();
JSONObject result = new JSONObject();
JSONArray elementsJson = new JSONArray();
try {
DisplayUtils.DisplayInfo displayInfo = DisplayUtils.getDisplayInfo(service);
List<A11yNode> rootSnapshots = captureRootSnapshots(service);
DumpStats stats = new DumpStats();
List<A11yNode> rootSnapshots = captureRootSnapshots(service, displayInfo, options, stats);
result.put("rotation", displayInfo.rotation);
result.put("width", displayInfo.width);
result.put("height", displayInfo.height);
if (rootSnapshots.isEmpty()) {
result.put("success", false);
result.put("error", "No active window or root node found");
result.put("xml", "");
result.put("elements", elementsJson);
result.put("rotation", displayInfo.rotation);
result.put("width", displayInfo.width);
result.put("height", displayInfo.height);
result.put("elements", new JSONArray());
return result;
}
// 1. Build standard UIAutomator XML
String xml = buildXml(rootSnapshots, displayInfo.rotation, displayInfo.width, displayInfo.height);
// 2. Build flat elements list
List<JSONObject> flatList = new ArrayList<>();
for (A11yNode root : rootSnapshots) {
root.collectFlatElements(flatList);
if (options.wantXml) {
result.put("xml", buildXml(rootSnapshots, displayInfo.rotation));
}
for (JSONObject elem : flatList) {
elementsJson.put(elem);
if (options.wantElements) {
JSONArray elementsJson = new JSONArray();
List<JSONObject> flatList = new ArrayList<>();
for (A11yNode root : rootSnapshots) {
root.collectFlatElements(flatList);
}
for (JSONObject elem : flatList) {
elementsJson.put(elem);
}
result.put("elements", elementsJson);
}
// 3. Build tree
JSONArray trees = new JSONArray();
for (A11yNode root : rootSnapshots) {
trees.put(root.toTreeJson());
if (options.wantTree) {
JSONArray trees = new JSONArray();
for (A11yNode root : rootSnapshots) {
trees.put(root.toTreeJson());
}
result.put("tree", trees.length() == 1 ? trees.getJSONObject(0) : trees);
}
long elapsed = System.currentTimeMillis() - startTime;
result.put("success", true);
result.put("xml", xml);
result.put("elements", elementsJson);
result.put("tree", trees.length() == 1 ? trees.getJSONObject(0) : trees);
result.put("rotation", displayInfo.rotation);
result.put("width", displayInfo.width);
result.put("height", displayInfo.height);
result.put("elapsed_ms", elapsed);
result.put("node_count", stats.nodes);
result.put("skipped_invisible", stats.skippedInvisible);
result.put("truncated", stats.truncated);
result.put("window_count", rootSnapshots.size());
result.put("elapsed_ms", System.currentTimeMillis() - startTime);
if (service instanceof ArtemisAccessibilityService) {
ArtemisAccessibilityService s = (ArtemisAccessibilityService) service;
@@ -114,70 +175,109 @@ public final class HierarchyDumper {
result.put("success", false);
result.put("error", "Dump failed: " + t.getMessage());
result.put("xml", "");
result.put("elements", elementsJson);
result.put("elements", new JSONArray());
} catch (Throwable ignored) {}
}
return result;
}
/**
* Dumps an atomic snapshot combining hardware screenshot (JPEG base64) and UI hierarchy XML/JSON
* at the exact same clock tick, eliminating temporal phase mismatch.
*/
private static final int ERROR_TAKE_SCREENSHOT_INTERVAL_TIME_SHORT = 3;
private static final long SCREENSHOT_INTERVAL_RETRY_MS = 350L;
@android.annotation.TargetApi(Build.VERSION_CODES.R)
private static void requestScreenshot(
final AccessibilityService service,
final AtomicReference<Bitmap> bitmapRef,
final AtomicInteger errorRef,
final CountDownLatch latch,
final boolean allowRetry) {
try {
service.takeScreenshot(
Display.DEFAULT_DISPLAY,
SCREENSHOT_EXECUTOR,
new AccessibilityService.TakeScreenshotCallback() {
@Override
public void onSuccess(AccessibilityService.ScreenshotResult screenshotResult) {
try {
HardwareBuffer buffer = screenshotResult.getHardwareBuffer();
ColorSpace colorSpace = screenshotResult.getColorSpace();
Bitmap hwBitmap = Bitmap.wrapHardwareBuffer(buffer, colorSpace);
if (hwBitmap != null) {
Bitmap swBitmap = hwBitmap.copy(Bitmap.Config.ARGB_8888, false);
hwBitmap.recycle();
buffer.close();
bitmapRef.set(swBitmap);
}
} catch (Throwable t) {
Log.w(TAG, "Error copying screenshot buffer", t);
} finally {
latch.countDown();
}
}
@Override
public void onFailure(int errorCode) {
errorRef.set(errorCode);
if (allowRetry && errorCode == ERROR_TAKE_SCREENSHOT_INTERVAL_TIME_SHORT) {
Log.i(TAG, "takeScreenshot rate-limited; retrying after " + SCREENSHOT_INTERVAL_RETRY_MS + " ms");
SCREENSHOT_EXECUTOR.execute(new Runnable() {
@Override
public void run() {
try {
Thread.sleep(SCREENSHOT_INTERVAL_RETRY_MS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
requestScreenshot(service, bitmapRef, errorRef, latch, false);
}
});
return;
}
Log.w(TAG, "takeScreenshot failed, errorCode: " + errorCode);
latch.countDown();
}
}
);
} catch (Throwable t) {
Log.w(TAG, "takeScreenshot invocation error", t);
latch.countDown();
}
}
public static JSONObject dumpAtomicSnapshot(AccessibilityService service) {
return dumpAtomicSnapshot(service, DumpOptions.forSnapshot());
}
/**
* Dumps an atomic snapshot combining hardware screenshot (JPEG base64) and UI hierarchy
* at the exact same clock tick, eliminating temporal phase mismatch. When the screenshot
* is present, {@code width} / {@code height} are the bitmap's own dimensions so the host
* normalizes coordinates against the very image it is looking at.
*/
public static JSONObject dumpAtomicSnapshot(AccessibilityService service, DumpOptions options) {
long startTime = System.currentTimeMillis();
// 1. Trigger hardware screenshot asynchronously
// 1. Trigger hardware screenshot asynchronously. The framework rate-limits
// takeScreenshot to one call per ~333 ms (ERROR_TAKE_SCREENSHOT_INTERVAL_TIME_SHORT = 3);
// back-to-back observations therefore wait out the interval and retry once
// here, which is far cheaper than the host falling back to adb screencap.
final AtomicReference<Bitmap> bitmapRef = new AtomicReference<>(null);
final AtomicInteger errorRef = new AtomicInteger(0);
final CountDownLatch screenshotLatch = new CountDownLatch(1);
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
try {
service.takeScreenshot(
Display.DEFAULT_DISPLAY,
SCREENSHOT_EXECUTOR,
new AccessibilityService.TakeScreenshotCallback() {
@Override
public void onSuccess(AccessibilityService.ScreenshotResult screenshotResult) {
try {
HardwareBuffer buffer = screenshotResult.getHardwareBuffer();
ColorSpace colorSpace = screenshotResult.getColorSpace();
Bitmap hwBitmap = Bitmap.wrapHardwareBuffer(buffer, colorSpace);
if (hwBitmap != null) {
Bitmap swBitmap = hwBitmap.copy(Bitmap.Config.ARGB_8888, false);
hwBitmap.recycle();
buffer.close();
bitmapRef.set(swBitmap);
}
} catch (Throwable t) {
Log.w(TAG, "Error copying screenshot buffer", t);
} finally {
screenshotLatch.countDown();
}
}
@Override
public void onFailure(int errorCode) {
Log.w(TAG, "takeScreenshot failed, errorCode: " + errorCode);
screenshotLatch.countDown();
}
}
);
} catch (Throwable t) {
Log.w(TAG, "takeScreenshot invocation error", t);
screenshotLatch.countDown();
}
requestScreenshot(service, bitmapRef, errorRef, screenshotLatch, true);
} else {
screenshotLatch.countDown();
}
// 2. Concurrently capture the UI hierarchy
JSONObject dumpData = dump(service);
JSONObject dumpData = dump(service, options);
// 3. Wait for the screenshot with a strict 1.5s timeout
// 3. Wait for the screenshot (one retry after the rate-limit interval fits inside)
try {
screenshotLatch.await(1500L, TimeUnit.MILLISECONDS);
screenshotLatch.await(2500L, TimeUnit.MILLISECONDS);
} catch (InterruptedException ignored) {}
Bitmap bitmap = bitmapRef.get();
@@ -189,6 +289,8 @@ public final class HierarchyDumper {
String base64Str = Base64.encodeToString(jpegBytes, Base64.NO_WRAP);
dumpData.put("screenshot_base64", base64Str);
dumpData.put("has_screenshot", true);
dumpData.put("width", bitmap.getWidth());
dumpData.put("height", bitmap.getHeight());
} catch (Throwable t) {
Log.w(TAG, "Failed to compress screenshot to JPEG Base64", t);
try { dumpData.put("has_screenshot", false); } catch (Throwable ignored) {}
@@ -198,8 +300,11 @@ public final class HierarchyDumper {
} else {
try {
dumpData.put("has_screenshot", false);
dumpData.put("screenshot_error_code", errorRef.get());
dumpData.put("screenshot_error", Build.VERSION.SDK_INT >= Build.VERSION_CODES.R
? "Screenshot capture timed out or failed"
? (errorRef.get() != 0
? "takeScreenshot failed with errorCode " + errorRef.get()
: "Screenshot capture timed out")
: "takeScreenshot not supported on Android < 11");
} catch (Throwable ignored) {}
}
@@ -211,23 +316,27 @@ public final class HierarchyDumper {
return dumpData;
}
public static String dumpXml(AccessibilityService service) {
return dumpXml(service, DumpOptions.forSnapshot());
}
/**
* Dumps the hierarchy directly as a raw standard UIAutomator XML string.
*/
public static String dumpXml(AccessibilityService service) {
public static String dumpXml(AccessibilityService service, DumpOptions options) {
DisplayUtils.DisplayInfo displayInfo = DisplayUtils.getDisplayInfo(service);
List<A11yNode> rootSnapshots = captureRootSnapshots(service);
List<A11yNode> rootSnapshots = captureRootSnapshots(service, displayInfo, options, new DumpStats());
if (rootSnapshots.isEmpty()) {
return "<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>\n<hierarchy rotation=\""
+ displayInfo.rotation + "\" />\n";
}
return buildXml(rootSnapshots, displayInfo.rotation, displayInfo.width, displayInfo.height);
return buildXml(rootSnapshots, displayInfo.rotation);
}
/**
* Builds the standard Android UIAutomator XML string from root snapshots.
*/
private static String buildXml(List<A11yNode> roots, int rotation, int width, int height) {
private static String buildXml(List<A11yNode> roots, int rotation) {
StringBuilder sb = new StringBuilder(roots.size() * 1024 + 256);
sb.append("<?xml version='1.0' encoding='UTF-8' standalone='yes' ?>\n");
sb.append("<hierarchy rotation=\"").append(rotation).append("\">");
@@ -249,6 +358,8 @@ public final class HierarchyDumper {
final int windowLayer;
final boolean windowActive;
final boolean windowFocused;
/** Screen rectangle of the window, or null when unknown (display bounds are used). */
final Rect windowBounds;
RawRootEntry(
AccessibilityNodeInfo root,
@@ -256,7 +367,8 @@ public final class HierarchyDumper {
String windowType,
int windowLayer,
boolean windowActive,
boolean windowFocused
boolean windowFocused,
Rect windowBounds
) {
this.root = root;
this.windowId = windowId;
@@ -264,14 +376,25 @@ public final class HierarchyDumper {
this.windowLayer = windowLayer;
this.windowActive = windowActive;
this.windowFocused = windowFocused;
this.windowBounds = windowBounds;
}
}
public static List<A11yNode> captureRootSnapshots(AccessibilityService service) {
return captureRootSnapshots(
service, DisplayUtils.getDisplayInfo(service), DumpOptions.forSnapshot(), new DumpStats());
}
/**
* Captures snapshots of all active and interactive windows with adaptive progressive retry support.
* Guaranteed to capture the complete hierarchy even during activity transitions and cold starts.
*/
public static List<A11yNode> captureRootSnapshots(AccessibilityService service) {
private static List<A11yNode> captureRootSnapshots(
AccessibilityService service,
DisplayUtils.DisplayInfo displayInfo,
DumpOptions options,
DumpStats stats
) {
List<RawRootEntry> rawRoots = Collections.emptyList();
long[] retryBackoff = new long[]{40L, 80L, 120L, 160L, 220L, 300L};
@@ -285,13 +408,19 @@ public final class HierarchyDumper {
}
}
Rect displayRect = new Rect(0, 0, displayInfo.width, displayInfo.height);
List<A11yNode> snapshots = new ArrayList<>(rawRoots.size());
int[] nodeCounter = new int[]{0};
for (int i = 0; i < rawRoots.size(); i++) {
RawRootEntry entry = rawRoots.get(i);
try {
A11yNode snapshot = snapshotNode(entry.root, 0, i, nodeCounter);
Rect clip = new Rect(displayRect);
if (entry.windowBounds != null && !clip.intersect(entry.windowBounds)) {
// Window entirely off the display (e.g. a second display): keep the
// display rect so the root still serializes with sane bounds.
clip = new Rect(displayRect);
}
A11yNode snapshot = snapshotNode(entry.root, 0, i, clip, options, stats);
if (snapshot != null) {
snapshot.windowId = entry.windowId;
snapshot.windowType = entry.windowType;
@@ -310,6 +439,24 @@ public final class HierarchyDumper {
return snapshots;
}
private static AccessibilityNodeInfo windowRoot(AccessibilityWindowInfo window) {
if (Build.VERSION.SDK_INT >= 33) {
try {
return window.getRoot(PREFETCH_DESCENDANTS_HYBRID);
} catch (Throwable ignored) {}
}
return window.getRoot();
}
private static AccessibilityNodeInfo childOf(AccessibilityNodeInfo node, int index) {
if (Build.VERSION.SDK_INT >= 33) {
try {
return node.getChild(index, PREFETCH_DESCENDANTS_HYBRID);
} catch (Throwable ignored) {}
}
return node.getChild(index);
}
/**
* Multi-tier hierarchy root discovery strategy:
* Tier 1: Multi-window enumeration (sorted by Z-layer descending).
@@ -335,7 +482,7 @@ public final class HierarchyDumper {
for (AccessibilityWindowInfo window : sortedWindows) {
try {
AccessibilityNodeInfo root = window.getRoot();
AccessibilityNodeInfo root = windowRoot(window);
if (root != null) {
int hash = root.hashCode();
if (seenHashes.add(hash)) {
@@ -343,13 +490,16 @@ public final class HierarchyDumper {
if (window.getType() == AccessibilityWindowInfo.TYPE_APPLICATION) {
hasAppWindow = true;
}
Rect bounds = new Rect();
window.getBoundsInScreen(bounds);
roots.add(new RawRootEntry(
root,
window.getId(),
typeStr,
window.getLayer(),
window.isActive(),
window.isFocused()
window.isFocused(),
bounds.isEmpty() ? null : bounds
));
} else {
safeRecycle(root);
@@ -373,7 +523,8 @@ public final class HierarchyDumper {
"application",
0,
true,
true
true,
null
));
hasAppWindow = true;
} else {
@@ -414,7 +565,8 @@ public final class HierarchyDumper {
"application",
0,
true,
true
true,
null
));
} else {
safeRecycle(current);
@@ -450,6 +602,11 @@ public final class HierarchyDumper {
* Recursively snapshots an AccessibilityNodeInfo into an immutable A11yNode,
* extracting rich semantic properties (error, heading, editable, paneTitle, tooltip, stateDescription).
*
* {@code clip} is the rectangle the node may be visible in: the display intersected with the
* window and every scrollable ancestor. Children that are not visible to the user are skipped
* unless {@code options.includeInvisible} is set; the window root itself is always kept, as in
* UIAutomator.
*
* Protected against recursion loops via MAX_DEPTH and MAX_NODES without flat hash collision drops.
*/
@SuppressWarnings("deprecation")
@@ -457,20 +614,37 @@ public final class HierarchyDumper {
AccessibilityNodeInfo node,
int depth,
int childIndex,
int[] nodeCounter
Rect clip,
DumpOptions options,
DumpStats stats
) {
if (node == null) return null;
if (depth > MAX_DEPTH || nodeCounter[0] >= MAX_NODES) {
if (depth > MAX_DEPTH || stats.nodes >= MAX_NODES) {
stats.truncated = true;
return null;
}
nodeCounter[0]++;
boolean visible = true;
try {
visible = node.isVisibleToUser();
} catch (Throwable ignored) {}
if (!visible && depth > 0 && !options.includeInvisible) {
stats.skippedInvisible++;
return null;
}
stats.nodes++;
A11yNode snapshot = new A11yNode();
snapshot.index = childIndex;
snapshot.visibleToUser = visible;
Rect childClip = clip;
try {
Rect bounds = new Rect();
node.getBoundsInScreen(bounds);
if (!bounds.intersect(clip)) {
bounds.setEmpty();
}
snapshot.left = bounds.left;
snapshot.top = bounds.top;
snapshot.right = bounds.right;
@@ -499,6 +673,13 @@ public final class HierarchyDumper {
snapshot.password = node.isPassword();
snapshot.selected = node.isSelected();
// Children of a scrollable container are clipped by it (UIAutomator's
// trimScrollableParent): a list row half under the toolbar keeps only its
// visible part, so its center is a point the user can actually touch.
if (snapshot.scrollable && !bounds.isEmpty()) {
childClip = new Rect(bounds);
}
// 1. Editable property
snapshot.editable = node.isEditable();
@@ -515,11 +696,15 @@ public final class HierarchyDumper {
} catch (Throwable ignored) {}
}
// 4. Hint text (API 26+)
// 4. Hint text (API 26+). When the field is empty the framework reports the
// hint as the text; keep text and hint apart so "empty input" stays detectable.
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
try {
CharSequence hint = node.getHintText();
if (hint != null) snapshot.hint = hint.toString();
if (node.isShowingHintText()) {
snapshot.text = "";
}
} catch (Throwable ignored) {}
}
@@ -557,9 +742,9 @@ public final class HierarchyDumper {
for (int i = 0; i < childCount; i++) {
AccessibilityNodeInfo childNode = null;
try {
childNode = node.getChild(i);
childNode = childOf(node, i);
if (childNode != null) {
A11yNode childSnapshot = snapshotNode(childNode, depth + 1, i, nodeCounter);
A11yNode childSnapshot = snapshotNode(childNode, depth + 1, i, childClip, options, stats);
if (childSnapshot != null) {
snapshot.children.add(childSnapshot);
}
@@ -594,34 +779,49 @@ public final class HierarchyDumper {
/**
* Finds the currently focused or editable input node for text entry.
* Window roots are enumerated once and recycled; only the returned node stays alive.
*/
public static AccessibilityNodeInfo findInputNode(AccessibilityService service) {
for (RawRootEntry entry : getActiveRawRoots(service)) {
try {
AccessibilityNodeInfo focused = entry.root.findFocus(AccessibilityNodeInfo.FOCUS_INPUT);
if (focused != null) return focused;
} catch (Throwable ignored) {}
List<RawRootEntry> roots = getActiveRawRoots(service);
AccessibilityNodeInfo found = null;
try {
for (RawRootEntry entry : roots) {
try {
AccessibilityNodeInfo focused = entry.root.findFocus(AccessibilityNodeInfo.FOCUS_INPUT);
if (focused != null) {
found = focused;
return found;
}
} catch (Throwable ignored) {}
}
for (RawRootEntry entry : roots) {
AccessibilityNodeInfo editable = findFirstEditable(entry.root, 0);
if (editable != null) {
found = editable;
return found;
}
}
return null;
} finally {
for (RawRootEntry entry : roots) {
if (entry.root != found) {
safeRecycle(entry.root);
}
}
}
for (RawRootEntry entry : getActiveRawRoots(service)) {
AccessibilityNodeInfo editable = findFirstEditable(entry.root);
if (editable != null) return editable;
}
return null;
}
private static AccessibilityNodeInfo findFirstEditable(AccessibilityNodeInfo node) {
if (node == null) return null;
private static AccessibilityNodeInfo findFirstEditable(AccessibilityNodeInfo node, int depth) {
if (node == null || depth > MAX_DEPTH) return null;
try {
if (node.isEditable() && node.isFocusable() && node.isEnabled()) {
if (node.isEditable() && node.isFocusable() && node.isEnabled() && node.isVisibleToUser()) {
return node;
}
int count = node.getChildCount();
for (int i = 0; i < count; i++) {
AccessibilityNodeInfo child = node.getChild(i);
AccessibilityNodeInfo child = childOf(node, i);
if (child != null) {
AccessibilityNodeInfo res = findFirstEditable(child);
AccessibilityNodeInfo res = findFirstEditable(child, depth + 1);
if (res != null) return res;
safeRecycle(child);
}
packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/TokenReceiver.java
+36
View File
@@ -0,0 +1,36 @@
package com.artemis.helper;
import android.content.BroadcastReceiver;
import android.content.Context;
import android.content.Intent;
import android.util.Log;
/**
* Receives the host's session token.
*
* The receiver is guarded in the manifest by
* {@code android.permission.WRITE_SECURE_SETTINGS}, a permission ordinary apps
* cannot hold but the adb shell user does, so only something with adb access to
* the device can set (or clear) the token:
*
* <pre>
* adb shell am broadcast -n com.artemis.helper/.TokenReceiver \
* -a com.artemis.helper.SET_TOKEN --es token &lt;hex&gt;
* </pre>
*/
public class TokenReceiver extends BroadcastReceiver {
private static final String TAG = "ArtemisTokenReceiver";
public static final String ACTION_SET_TOKEN = "com.artemis.helper.SET_TOKEN";
public static final String EXTRA_TOKEN = "token";
@Override
public void onReceive(Context context, Intent intent) {
if (intent == null || !ACTION_SET_TOKEN.equals(intent.getAction())) {
return;
}
String value = intent.getStringExtra(EXTRA_TOKEN);
TokenStore.set(value);
Log.i(TAG, TokenStore.isSet() ? "Session token set" : "Session token cleared");
}
}
packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper/TokenStore.java
+38
View File
@@ -0,0 +1,38 @@
package com.artemis.helper;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
/**
* The session token the host pushed for this process lifetime.
*
* The loopback server is reachable by every app on the device, so requests are
* only served when they carry this token. It lives in memory only: a killed or
* re-bound service starts without one and answers 401 until the host pushes it
* again (which the host does on every attach and on every 401).
*/
public final class TokenStore {
private static volatile String token;
private TokenStore() {}
public static void set(String value) {
token = (value == null || value.isEmpty()) ? null : value;
}
public static boolean isSet() {
return token != null;
}
/** Constant-time comparison so a local app cannot guess the token byte by byte. */
public static boolean matches(String candidate) {
String current = token;
if (current == null || candidate == null) {
return false;
}
return MessageDigest.isEqual(
current.getBytes(StandardCharsets.UTF_8),
candidate.getBytes(StandardCharsets.UTF_8));
}
}
packages/artemis-accessibility-helper/app/src/main/res/xml/accessibility_service_config.xml
+3 -3
View File
@@ -1,9 +1,9 @@
<?xml version="1.0" encoding="utf-8"?>
<accessibility-service xmlns:android="http://schemas.android.com/apk/res/android"
android:accessibilityEventTypes="typeAllMask"
android:accessibilityEventTypes="typeWindowStateChanged"
android:accessibilityFeedbackType="feedbackGeneric"
android:notificationTimeout="50"
android:accessibilityFlags="flagDefault|flagRetrieveInteractiveWindows|flagIncludeNotImportantViews|flagReportViewIds"
android:notificationTimeout="100"
android:accessibilityFlags="flagDefault|flagRetrieveInteractiveWindows|flagReportViewIds"
android:canRetrieveWindowContent="true"
android:canPerformGestures="true"
android:canTakeScreenshot="true"
packages/artemis-accessibility-helper/build_apk.sh
+114 -17
View File
@@ -1,22 +1,49 @@
#!/usr/bin/env bash
# Standalone build for ArtemisAccessibilityHelper.apk (no Gradle daemon).
#
# Outputs, next to this script:
# ArtemisAccessibilityHelper.apk the signed, zipaligned artifact
# helper_manifest.json {version_code, version_name, sha256, built_at}
#
# The Python runtime (artemis/runtime/helper_manager.py) reads helper_manifest.json
# to decide whether a device needs an install or an upgrade, so the manifest must
# always be regenerated together with the APK. The version is sourced from
# app/build.gradle.kts (defaultConfig.versionCode / versionName) so the Gradle
# build and this script cannot drift.
set -e
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
cd "$DIR"
_to_posix() {
if command -v cygpath >/dev/null 2>&1; then
cygpath -u "$1"
else
printf '%s' "$1"
fi
}
# 1. Locate Android SDK
SDK_ROOT="${ANDROID_HOME:-${ANDROID_SDK_ROOT:-$HOME/Library/Android/sdk}}"
if [ ! -d "$SDK_ROOT" ]; then
for candidate in /opt/android-sdk /usr/local/share/android-sdk "$HOME/Android/Sdk"; do
if [ -d "$candidate" ]; then
SDK_ROOT="${ANDROID_HOME:-${ANDROID_SDK_ROOT:-}}"
if [ -n "$SDK_ROOT" ]; then
SDK_ROOT="$(_to_posix "$SDK_ROOT")"
fi
if [ -z "$SDK_ROOT" ] || [ ! -d "$SDK_ROOT" ]; then
for candidate in \
"$HOME/Library/Android/sdk" \
"$HOME/Android/Sdk" \
/opt/android-sdk \
/usr/local/share/android-sdk \
"$( [ -n "$LOCALAPPDATA" ] && _to_posix "$LOCALAPPDATA/Android/Sdk" )"; do
if [ -n "$candidate" ] && [ -d "$candidate" ]; then
SDK_ROOT="$candidate"
break
fi
done
fi
if [ ! -d "$SDK_ROOT" ]; then
echo "Error: Android SDK not found at $SDK_ROOT. Set ANDROID_HOME." >&2
if [ -z "$SDK_ROOT" ] || [ ! -d "$SDK_ROOT" ]; then
echo "Error: Android SDK not found. Set ANDROID_HOME." >&2
exit 1
fi
@@ -35,9 +62,15 @@ fi
# 2. Locate Java compiler and runtime
if [ -z "$JAVA_HOME" ]; then
if [ -d "/Applications/Android Studio.app/Contents/jbr/Contents/Home" ]; then
export JAVA_HOME="/Applications/Android Studio.app/Contents/jbr/Contents/Home"
fi
for candidate in \
"/Applications/Android Studio.app/Contents/jbr/Contents/Home" \
"$(_to_posix "${PROGRAMFILES:-/c/Program Files}/Android/Android Studio/jbr")" \
"/opt/android-studio/jbr"; do
if [ -n "$candidate" ] && [ -x "$candidate/bin/javac" ] || [ -x "$candidate/bin/javac.exe" ]; then
export JAVA_HOME="$candidate"
break
fi
done
fi
if [ -n "$JAVA_HOME" ]; then
export PATH="$JAVA_HOME/bin:$PATH"
@@ -48,8 +81,49 @@ if ! command -v javac >/dev/null 2>&1; then
exit 1
fi
PYTHON_BIN="${PYTHON:-}"
if [ -z "$PYTHON_BIN" ]; then
# The Windows Store "python" stub is on PATH but exits 49; prefer the project venv.
for candidate in "$DIR/../../.venv/Scripts/python.exe" "$DIR/../../.venv/bin/python" python3 python; do
if "$candidate" -c "import zipfile" >/dev/null 2>&1; then
PYTHON_BIN="$candidate"
break
fi
done
fi
if [ -z "$PYTHON_BIN" ]; then
echo "Error: python3 not found (needed to package the DEX and write helper_manifest.json)" >&2
exit 1
fi
# Build-tools ship as .exe / .bat wrappers on Windows; resolve the right name once.
_tool() {
local base="$BUILD_TOOLS/$1"
for suffix in "" ".exe" ".bat"; do
if [ -f "$base$suffix" ]; then
printf '%s' "$base$suffix"
return 0
fi
done
echo "Error: $1 not found under $BUILD_TOOLS" >&2
exit 1
}
AAPT2="$(_tool aapt2)"
D8="$(_tool d8)"
ZIPALIGN="$(_tool zipalign)"
APKSIGNER="$(_tool apksigner)"
# 3. Version: single source is app/build.gradle.kts
VERSION_CODE="$(sed -n 's/^[[:space:]]*versionCode[[:space:]]*=[[:space:]]*\([0-9][0-9]*\).*/\1/p' app/build.gradle.kts | head -n 1)"
VERSION_NAME="$(sed -n 's/^[[:space:]]*versionName[[:space:]]*=[[:space:]]*"\([^"]*\)".*/\1/p' app/build.gradle.kts | head -n 1)"
if [ -z "$VERSION_CODE" ] || [ -z "$VERSION_NAME" ]; then
echo "Error: could not read versionCode / versionName from app/build.gradle.kts" >&2
exit 1
fi
echo "=================================================="
echo " Building ArtemisAccessibilityHelper.apk"
echo " Version: $VERSION_NAME (code $VERSION_CODE)"
echo " Build Tools: $(basename "$BUILD_TOOLS")"
echo " Platform: $(basename "$PLATFORM")"
echo " Java: $(javac -version 2>&1)"
@@ -59,14 +133,16 @@ rm -rf build
mkdir -p build/gen build/obj build/apk build/res_compiled
echo "-> Compiling resources..."
"$BUILD_TOOLS/aapt2" compile --dir app/src/main/res -o build/res_compiled/res.zip
"$AAPT2" compile --dir app/src/main/res -o build/res_compiled/res.zip
echo "-> Linking APK..."
"$BUILD_TOOLS/aapt2" link \
"$AAPT2" link \
-I "$PLATFORM/android.jar" \
--manifest app/src/main/AndroidManifest.xml \
--min-sdk-version 24 \
--target-sdk-version 35 \
--version-code "$VERSION_CODE" \
--version-name "$VERSION_NAME" \
--java build/gen \
-o build/apk/unaligned.apk \
build/res_compiled/res.zip
@@ -74,25 +150,29 @@ echo "-> Linking APK..."
echo "-> Compiling Java sources..."
JAVA_FILES="$(find build/gen app/src/main/java -name "*.java")"
javac -encoding UTF-8 \
-source 8 -target 8 \
-Xlint:-options \
-cp "$PLATFORM/android.jar" \
-d build/obj \
$JAVA_FILES
echo "-> Dexing with D8..."
CLASS_FILES="$(find build/obj -name "*.class")"
"$BUILD_TOOLS/d8" \
"$D8" \
--lib "$PLATFORM/android.jar" \
--output build/apk/ \
--min-api 24 \
$CLASS_FILES
echo "-> Packaging DEX..."
cd build/apk
zip -u -q unaligned.apk classes.dex
cd "$DIR"
"$PYTHON_BIN" - <<'PY'
import zipfile
with zipfile.ZipFile("build/apk/unaligned.apk", "a", compression=zipfile.ZIP_DEFLATED) as apk:
apk.write("build/apk/classes.dex", "classes.dex")
PY
echo "-> Zipaligning..."
"$BUILD_TOOLS/zipalign" -f -p 4 build/apk/unaligned.apk build/apk/aligned.apk
"$ZIPALIGN" -f -p 4 build/apk/unaligned.apk build/apk/aligned.apk
echo "-> Signing APK..."
if [ ! -f debug.keystore ]; then
@@ -103,7 +183,7 @@ if [ ! -f debug.keystore ]; then
-dname "CN=Artemis Debug,O=Artemis,C=US"
fi
"$BUILD_TOOLS/apksigner" sign \
"$APKSIGNER" sign \
--ks debug.keystore \
--ks-pass pass:android \
--key-pass pass:android \
@@ -111,6 +191,23 @@ fi
--out ArtemisAccessibilityHelper.apk \
build/apk/aligned.apk
echo "-> Writing helper_manifest.json..."
VERSION_CODE="$VERSION_CODE" VERSION_NAME="$VERSION_NAME" "$PYTHON_BIN" - <<'PY'
import datetime, hashlib, json, os
digest = hashlib.sha256(open("ArtemisAccessibilityHelper.apk", "rb").read()).hexdigest()
manifest = {
"package": "com.artemis.helper",
"version_code": int(os.environ["VERSION_CODE"]),
"version_name": os.environ["VERSION_NAME"],
"sha256": digest,
"built_at": datetime.datetime.now(datetime.timezone.utc).replace(microsecond=0).isoformat(),
}
with open("helper_manifest.json", "w", encoding="utf-8", newline="\n") as fh:
json.dump(manifest, fh, indent=2)
fh.write("\n")
print(json.dumps(manifest, indent=2))
PY
echo "=================================================="
echo " Build SUCCESSFUL!"
echo " Artifact: $DIR/ArtemisAccessibilityHelper.apk"
packages/artemis-accessibility-helper/helper_manifest.json
+7
View File
@@ -0,0 +1,7 @@
{
"package": "com.artemis.helper",
"version_code": 6,
"version_name": "1.2.0",
"sha256": "133e373fe03615c251cd74b03a38539c844d09a074954bca2d2dde6684561969",
"built_at": "2026-09-10T23:13:11+00:00"
}