feat(android): add native controls and continuous multi-touch

This commit is contained in:
chenmingxuan committed 2026-11-02 21:33:45 +08:00
1 parent 3dd96078db
commit 4b12ae7728
70 files changed
+13105 -179

No files matched your search

@@ -10,8 +10,8 @@ android {
applicationId = "com.artemis.helper"
minSdk = 24
targetSdk = 35
versionCode = 9
versionName = "1.2.3"
versionCode = 15
versionName = "1.3.0"
}
buildTypes {
@@ -31,5 +31,5 @@ android {
dependencies {
// Pure standard Android SDK APIs (android.accessibilityservice, android.view.accessibility, org.json)
// Zero external dependencies to maximize stability, guarantee compatibility, and keep APK under 30KB.
// Zero external dependencies; keep the standalone APK small and compatible.
}
@@ -4,6 +4,7 @@ import org.json.JSONArray;
import org.json.JSONObject;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
/**
@@ -55,6 +56,9 @@ public final class A11yNode {
public String paneTitle = "";
public String tooltip = "";
// Detached cheap native semantics; never retains AccessibilityNodeInfo objects.
public JSONObject nativeMetadata = new JSONObject();
public int left = 0;
public int top = 0;
public int right = 0;
@@ -100,6 +104,7 @@ public final class A11yNode {
XmlUtils.appendBooleanAttribute(sb, "scrollable", scrollable);
XmlUtils.appendBooleanAttribute(sb, "long-clickable", longClickable);
XmlUtils.appendBooleanAttribute(sb, "password", password);
XmlUtils.appendBooleanAttribute(sb, "editable", editable);
XmlUtils.appendBooleanAttribute(sb, "selected", selected);
XmlUtils.appendBooleanAttribute(sb, "visible-to-user", visibleToUser);
XmlUtils.appendAttribute(sb, "bounds", getBoundsString());
@@ -110,6 +115,20 @@ public final class A11yNode {
if (showingHint != null) {
XmlUtils.appendBooleanAttribute(sb, "showing-hint-text", showingHint);
}
JSONObject range = nativeMetadata.optJSONObject("range");
if (range != null) {
XmlUtils.appendAttribute(sb, "range", range.toString());
}
JSONArray actions = nativeMetadata.optJSONArray("actions");
if (actions != null) {
XmlUtils.appendAttribute(sb, "actions", actions.toString());
}
if (nativeMetadata.has("selection_start")) {
XmlUtils.appendIntAttribute(sb, "selection-start", nativeMetadata.optInt("selection_start", -1));
}
if (nativeMetadata.has("selection_end")) {
XmlUtils.appendIntAttribute(sb, "selection-end", nativeMetadata.optInt("selection_end", -1));
}
// Window metadata (emitted on window root nodes)
if (windowId >= 0) {
@@ -127,9 +146,6 @@ public final class A11yNode {
}
// Extended semantic attributes
if (editable) {
XmlUtils.appendBooleanAttribute(sb, "editable", true);
}
if (isHeading) {
XmlUtils.appendBooleanAttribute(sb, "heading", true);
}
@@ -252,6 +268,13 @@ public final class A11yNode {
if (!tooltip.isEmpty()) {
obj.put("tooltip", tooltip);
}
// Copies keep flat/tree consumers from sharing mutable snapshot values.
JSONObject metadata = new JSONObject(nativeMetadata.toString());
Iterator<String> keys = metadata.keys();
while (keys.hasNext()) {
String key = keys.next();
obj.put(key, metadata.get(key));
}
} catch (Throwable ignored) {}
return obj;
}
@@ -264,7 +287,10 @@ public final class A11yNode {
boolean hasContent = !text.isEmpty() || !contentDesc.isEmpty() || !resourceId.isEmpty()
|| !stateDescription.isEmpty() || !errorText.isEmpty() || !paneTitle.isEmpty();
boolean isInteractive = clickable || scrollable || checkable || focusable || longClickable || editable;
return hasContent || isInteractive || isHeading;
JSONArray actions = nativeMetadata.optJSONArray("actions");
boolean hasNativeControls = nativeMetadata.optJSONObject("range") != null
|| (actions != null && actions.length() > 0);
return hasContent || isInteractive || isHeading || hasNativeControls;
}
/**
@@ -31,7 +31,7 @@ public class ArtemisAccessibilityService extends AccessibilityService {
* Bumped whenever the HTTP contract changes in a way an older host cannot use.
* 2: token authentication, byte-accurate bodies, visible-only dumps, fields=.
*/
public static final int PROTOCOL_VERSION = 2;
public static final int PROTOCOL_VERSION = 3;
// Channel importance is frozen by Android once a channel exists, so a quieter
// channel needs a new id; the pre-1.1.3 channel is deleted on start.
private static final String LEGACY_CHANNEL_ID = "artemis_helper_channel";
@@ -294,6 +294,14 @@ public class CommandServer extends Thread {
JSONObject resp = new JSONObject();
try {
switch (cmd.toLowerCase(Locale.ROOT)) {
case "inspect_control":
return NativeControls.inspect(service, params);
case "control_action":
return NativeControls.act(service, params);
case "touch_gesture":
return TouchGestures.perform(service, params);
case "cancel_gesture":
return TouchGestures.cancel(params);
case "ping":
return buildPing(true);
case "dump":
@@ -388,6 +396,10 @@ public class CommandServer extends Thread {
r.put("version_code", service.getVersionCode());
r.put("version_name", service.getVersionName());
r.put("protocol_version", ArtemisAccessibilityService.PROTOCOL_VERSION);
org.json.JSONArray capabilities = new org.json.JSONArray();
capabilities.put("inspect_control").put("control_action");
if (android.os.Build.VERSION.SDK_INT >= 26) capabilities.put("touch_gesture").put("cancel_gesture");
r.put("capabilities", capabilities);
r.put("port", port);
r.put("auth_required", true);
r.put("token_set", TokenStore.isSet());
@@ -0,0 +1,263 @@
package com.artemis.helper;
import android.accessibilityservice.AccessibilityService;
import android.graphics.Rect;
import android.graphics.Region;
import android.os.Build;
import android.view.accessibility.AccessibilityNodeInfo;
import android.view.accessibility.AccessibilityWindowInfo;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.IdentityHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicBoolean;
/** Owns a conservative, bounded set of native-control roots and all acquired handles. */
final class ControlWindowRoots implements AutoCloseable {
private static final int MAX_WINDOWS = 64;
private final List<AccessibilityNodeInfo> roots = new ArrayList<>();
String error;
String message;
private ControlWindowRoots() {}
static ControlWindowRoots capture(AccessibilityService service, AtomicBoolean cancelled) {
ControlWindowRoots result = new ControlWindowRoots();
List<AccessibilityWindowInfo> windows = null;
try {
if (cancelled.get()) {
result.fail("CANCELLED", "The caller stopped waiting during window discovery");
return result;
}
AccessibilityNodeInfo active = service.getRootInActiveWindow();
if (active == null) {
result.fail("UNAVAILABLE", "There is no active window root");
return result;
}
result.roots.add(active);
final int activeId;
final String packageName;
try {
activeId = active.getWindowId();
CharSequence name = active.getPackageName();
packageName = name == null ? "" : name.toString();
} catch (Throwable unavailable) {
result.fail("UNAVAILABLE", "The active window identity could not be established");
return result;
}
if (activeId < 0 || packageName.isEmpty()) {
result.fail("UNAVAILABLE", "The active window identity could not be established");
return result;
}
try {
windows = service.getWindows();
} catch (Throwable unavailable) {
result.fail("UNAVAILABLE", "Native window enumeration is unavailable");
return result;
}
if (windows == null || windows.isEmpty()) {
result.fail("UNAVAILABLE", "Native window enumeration is incomplete");
return result;
}
if (windows.size() > MAX_WINDOWS) {
result.fail("LIMIT_EXCEEDED", "Window discovery exceeds the bounded search limit");
return result;
}
AccessibilityWindowInfo activeWindow = null;
int activeLayer = 0;
int activeDisplay = 0;
try {
for (AccessibilityWindowInfo window : windows) {
if (window != null && window.getId() == activeId) {
if (window.getType() != AccessibilityWindowInfo.TYPE_APPLICATION
|| !window.isActive()) {
result.fail("UNAVAILABLE", "Active application window metadata is unavailable");
return result;
}
int layer = window.getLayer();
int display = displayId(window);
if (activeWindow != null && (layer != activeLayer || display != activeDisplay)) {
result.fail("UNAVAILABLE", "Active window metadata is inconsistent");
return result;
}
activeWindow = window;
activeLayer = layer;
activeDisplay = display;
}
}
} catch (Throwable unavailable) {
result.fail("UNAVAILABLE", "Active window metadata could not be read");
return result;
}
if (activeWindow == null) {
result.fail("UNAVAILABLE", "The active root has no matching window metadata");
return result;
}
for (AccessibilityWindowInfo window : windows) {
if (cancelled.get()) {
result.fail("CANCELLED", "The caller stopped waiting during window discovery");
break;
}
if (window == null) {
result.fail("READ_FAILED", "A declared window could not be read");
break;
}
// The active root was already acquired; windows below its modal layer stay excluded.
if (window.getId() == activeId
|| window.getType() != AccessibilityWindowInfo.TYPE_APPLICATION
|| window.getLayer() < activeLayer || displayId(window) != activeDisplay) continue;
AccessibilityNodeInfo root = null;
try {
root = window.getRoot();
if (root == null) {
result.fail("READ_FAILED", "An eligible application window has no readable root");
break;
}
CharSequence name = root.getPackageName();
if (name == null || name.toString().isEmpty()) {
result.fail("READ_FAILED", "An application window package could not be established");
break;
}
if (!packageName.equals(name.toString())) continue;
if (root.getWindowId() != window.getId()) {
result.fail("READ_FAILED", "An application window changed during discovery");
break;
}
if (result.add(root)) root = null;
} finally {
recycle(root);
}
}
} catch (Throwable unavailable) {
result.fail("READ_FAILED", "The native windows changed during discovery");
} finally {
recycleWindows(windows);
}
return result;
}
static final class Blocked {
final String code;
final String message;
Blocked(String code, String message) { this.code = code; this.message = message; }
}
/** Recheck every native write against the target area left exposed inside its own window. */
static Blocked mutationBlock(AccessibilityService service, AccessibilityNodeInfo node) {
List<AccessibilityWindowInfo> windows = null;
try {
if (!node.isEnabled()) return new Blocked("DISABLED", "The native target is disabled");
if (!node.isVisibleToUser()) {
return new Blocked("NOT_VISIBLE", "The native target is not visible");
}
Rect target = new Rect();
node.getBoundsInScreen(target);
if (target.isEmpty()) return new Blocked("UNAVAILABLE", "The target has no usable bounds");
windows = service.getWindows();
if (windows == null || windows.isEmpty() || windows.size() > MAX_WINDOWS) {
return new Blocked("UNAVAILABLE", "Current window occlusion cannot be established");
}
Map<Integer, WindowState> states = new HashMap<>();
for (AccessibilityWindowInfo window : windows) {
if (window == null) return new Blocked("UNAVAILABLE", "Current window metadata is incomplete");
WindowState state = new WindowState(window);
WindowState previous = states.put(state.id, state);
if (previous != null && !previous.same(state)) {
return new Blocked("UNAVAILABLE", "Current window metadata is inconsistent");
}
}
WindowState own = states.get(node.getWindowId());
if (own == null || own.bounds.isEmpty() || !Rect.intersects(own.bounds, target)) {
return new Blocked("UNAVAILABLE", "The target window and bounds cannot be established");
}
Region exposed = new Region(target);
exposed.op(own.region, Region.Op.INTERSECT);
for (WindowState state : states.values()) {
if (state.id != own.id && state.display == own.display && state.layer > own.layer
&& !state.region.isEmpty()) {
exposed.op(state.region, Region.Op.DIFFERENCE);
}
}
if (exposed.isEmpty()) {
return new Blocked("OCCLUDED", "No target area remains exposed inside its window");
}
return null;
} catch (Throwable unavailable) {
return new Blocked("UNAVAILABLE", "Current target visibility or window metadata could not be read");
} finally {
recycleWindows(windows);
}
}
private static final class WindowState {
final int id;
final int layer;
final int display;
final Rect bounds = new Rect();
final Region region = new Region();
WindowState(AccessibilityWindowInfo window) {
id = window.getId();
layer = window.getLayer();
display = displayId(window);
window.getBoundsInScreen(bounds);
if (Build.VERSION.SDK_INT >= 30) window.getRegionInScreen(region);
else region.set(bounds);
if (id < 0) throw new IllegalArgumentException("Unknown window identity");
}
boolean same(WindowState other) {
return id == other.id && layer == other.layer && display == other.display
&& bounds.equals(other.bounds) && region.equals(other.region);
}
}
@SuppressWarnings("deprecation")
private static void recycleWindows(List<AccessibilityWindowInfo> windows) {
if (windows == null) return;
IdentityHashMap<AccessibilityWindowInfo, Boolean> recycled = new IdentityHashMap<>();
for (AccessibilityWindowInfo window : windows) {
if (window != null && recycled.put(window, Boolean.TRUE) == null) {
try { window.recycle(); } catch (Throwable ignored) {}
}
}
}
private static int displayId(AccessibilityWindowInfo window) {
int display = Build.VERSION.SDK_INT >= 30 ? window.getDisplayId() : 0;
if (display < 0) throw new IllegalArgumentException("Unknown window display");
return display;
}
/** Returns true when this collection owns the handle, including an identical duplicate. */
private boolean add(AccessibilityNodeInfo root) {
for (AccessibilityNodeInfo existing : roots) {
if (existing.equals(root)) return existing == root;
}
roots.add(root);
return true;
}
private void fail(String code, String detail) {
error = code;
message = detail;
}
int size() { return roots.size(); }
AccessibilityNodeInfo take(int index) {
return roots.set(index, null);
}
@Override public void close() {
for (AccessibilityNodeInfo root : roots) recycle(root);
roots.clear();
}
@SuppressWarnings("deprecation")
private static void recycle(AccessibilityNodeInfo root) {
if (root != null) {
try { root.recycle(); } catch (Throwable ignored) {}
}
}
}
@@ -10,6 +10,7 @@ import android.os.Build;
import android.os.Bundle;
import android.os.Handler;
import android.os.Looper;
import android.os.SystemClock;
import android.util.Log;
import android.view.accessibility.AccessibilityNodeInfo;
@@ -37,11 +38,18 @@ public final class GestureController {
}
public static boolean doubleTap(AccessibilityService service, float x, float y, long timeoutMs) {
if (!tap(service, x, y, timeoutMs)) return false;
try {
Thread.sleep(100L);
} catch (InterruptedException ignored) {}
return tap(service, x, y, timeoutMs);
if (timeoutMs <= 0) return false;
Path first = new Path();
first.moveTo(x, y);
Path second = new Path();
second.moveTo(x, y);
GestureDescription gesture = new GestureDescription.Builder()
.addStroke(new GestureDescription.StrokeDescription(first, 0L, 60L))
.addStroke(new GestureDescription.StrokeDescription(second, 160L, 60L))
.build();
long waitMs = timeoutMs > (Long.MAX_VALUE - 100L) / 2L
? Long.MAX_VALUE : timeoutMs * 2L + 100L;
return dispatchSynchronous(service, gesture, waitMs);
}
public static boolean longPress(AccessibilityService service, float x, float y, long durationMs, long timeoutMs) {
@@ -162,38 +170,102 @@ public final class GestureController {
final GestureDescription gesture,
long timeoutMs
) {
final CountDownLatch latch = new CountDownLatch(1);
final AtomicBoolean result = new AtomicBoolean(false);
if (service == null || timeoutMs <= 0 || Thread.currentThread().isInterrupted()
|| Looper.myLooper() == Looper.getMainLooper()) {
return false;
}
final LegacyDispatch pending = new LegacyDispatch(service, gesture, timeoutMs);
if (!TouchGestures.claimLegacy(pending)) return false;
if (!MAIN_HANDLER.post(pending.dispatch)) {
pending.finish(false);
return false;
}
try {
long remaining = pending.deadlineAt - SystemClock.uptimeMillis();
if (remaining > 0 && pending.done.await(remaining, TimeUnit.MILLISECONDS)) {
return pending.succeeded();
}
return false;
} catch (InterruptedException interrupted) {
Thread.currentThread().interrupt();
return false;
} finally {
pending.abandon();
}
}
MAIN_HANDLER.post(new Runnable() {
@Override
public void run() {
/** A timed-out queued dispatch is abandoned; uncertain input retains ownership. */
private static final class LegacyDispatch {
final AccessibilityService service;
final GestureDescription gesture;
final long deadlineAt;
final CountDownLatch done = new CountDownLatch(1);
boolean attempted;
boolean abandoned;
boolean ended;
boolean completed;
LegacyDispatch(AccessibilityService service, GestureDescription gesture, long timeoutMs) {
this.service = service;
this.gesture = gesture;
long now = SystemClock.uptimeMillis();
deadlineAt = timeoutMs > Long.MAX_VALUE - now ? Long.MAX_VALUE : now + timeoutMs;
}
final Runnable dispatch = new Runnable() {
@Override public void run() {
synchronized (LegacyDispatch.this) {
if (ended || abandoned || SystemClock.uptimeMillis() >= deadlineAt) {
finish(false);
return;
}
attempted = true;
}
try {
service.dispatchGesture(gesture, new AccessibilityService.GestureResultCallback() {
@Override
public void onCompleted(GestureDescription gestureDescription) {
result.set(true);
latch.countDown();
}
@Override
public void onCancelled(GestureDescription gestureDescription) {
result.set(false);
latch.countDown();
}
}, null);
} catch (Throwable t) {
Log.w(TAG, "dispatchGesture threw exception", t);
result.set(false);
latch.countDown();
boolean accepted = service.dispatchGesture(gesture,
new AccessibilityService.GestureResultCallback() {
@Override public void onCompleted(GestureDescription description) {
finish(true);
}
@Override public void onCancelled(GestureDescription description) {
finish(false);
}
}, MAIN_HANDLER);
if (!accepted) finish(false);
} catch (Throwable uncertain) {
Log.w(TAG, "dispatchGesture completion is unknown", uncertain);
synchronized (LegacyDispatch.this) { abandoned = true; }
// A late callback may release this owner; never assume thrown
// dispatch is quiet and let a subsequent request inject input.
done.countDown();
}
}
});
};
try {
return latch.await(timeoutMs, TimeUnit.MILLISECONDS) && result.get();
} catch (InterruptedException e) {
return false;
synchronized boolean succeeded() {
return ended && completed && !abandoned;
}
void abandon() {
synchronized (this) {
abandoned = true;
if (!attempted && !ended) {
ended = true;
TouchGestures.releaseLegacy(this);
done.countDown();
}
}
MAIN_HANDLER.removeCallbacks(dispatch);
}
void finish(boolean success) {
synchronized (this) {
if (ended) return;
ended = true;
completed = success;
}
TouchGestures.releaseLegacy(this);
done.countDown();
}
}
@@ -677,9 +677,18 @@ public final class HierarchyDumper {
snapshot.focused = node.isFocused();
snapshot.scrollable = node.isScrollable();
snapshot.longClickable = node.isLongClickable();
snapshot.password = node.isPassword();
snapshot.selected = node.isSelected();
// One shared, cheap native mapping for hierarchy and explicit control reads.
// Copy the JSON before recycling this live node; no descendant walk or token.
try {
snapshot.nativeMetadata = new JSONObject(NativeControls.metadata(node).toString());
snapshot.password = snapshot.nativeMetadata.optBoolean("password", false);
snapshot.editable = snapshot.nativeMetadata.optBoolean("editable", false);
} catch (Throwable unavailable) {
Log.w(TAG, "Native control metadata unavailable", unavailable);
}
// Children of a scrollable container are clipped by it (UIAutomator's
// trimScrollableParent): a list row half under the toolbar keeps only its
// visible part, so its center is a point the user can actually touch.
@@ -687,9 +696,6 @@ public final class HierarchyDumper {
childClip = new Rect(bounds);
}
// 1. Editable property
snapshot.editable = node.isEditable();
// 2. Error message (crucial for form validation detection)
CharSequence err = node.getError();
if (err != null && err.length() > 0) {
@@ -0,0 +1,688 @@
package com.artemis.helper;
import android.accessibilityservice.AccessibilityService;
import android.graphics.Rect;
import android.os.Build;
import android.os.Bundle;
import android.os.Handler;
import android.os.Looper;
import android.view.accessibility.AccessibilityNodeInfo;
import org.json.JSONArray;
import org.json.JSONObject;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
/** Exact, bounded native node operations. No coordinate or alternate-node fallback. */
public final class NativeControls {
private static final Handler MAIN = new Handler(Looper.getMainLooper());
private static final int MAX_DEPTH = 75;
private static final int MAX_NODES = 6000;
private static final int MAX_CANDIDATES = 10;
private static final long MAIN_WAIT_MS = 4000L;
private static final String[] ACTION_NAMES = {
"click", "long_click", "focus", "show_on_screen", "set_progress",
"scroll_forward", "scroll_backward", "scroll_up", "scroll_down",
"scroll_left", "scroll_right", "set_selection", "set_text", "ime_enter"
};
private NativeControls() {}
public static JSONObject inspect(final AccessibilityService service, final JSONObject args) {
return onMain(service, args, false);
}
public static JSONObject act(final AccessibilityService service, final JSONObject args) {
JSONObject result = onMain(service, args, true);
return result.has("node_before") ? result : withNodes(result, null, null);
}
/** A queued operation can be abandoned safely; a running write has unknown completion. */
private static JSONObject onMain(
final AccessibilityService service, final JSONObject args, final boolean mutation) {
if (service == null || args == null) {
return failure("INVALID_ARGUMENT", "A service and arguments are required", "not_started");
}
if (Looper.myLooper() == Looper.getMainLooper()) {
return execute(service, args, mutation, new AtomicBoolean(false));
}
final CountDownLatch done = new CountDownLatch(1);
final AtomicInteger state = new AtomicInteger(0); // queued, running, done, abandoned
final AtomicBoolean cancelled = new AtomicBoolean(false);
final AtomicReference<JSONObject> result = new AtomicReference<>();
final Runnable work = new Runnable() {
@Override public void run() {
if (!state.compareAndSet(0, 1)) {
done.countDown();
return;
}
try {
result.set(execute(service, args, mutation, cancelled));
} finally {
state.set(2);
done.countDown();
}
}
};
if (!MAIN.post(work)) {
return failure("UNAVAILABLE", "The service main thread is unavailable", "not_started");
}
try {
if (done.await(MAIN_WAIT_MS, TimeUnit.MILLISECONDS) && result.get() != null) {
return result.get();
}
} catch (InterruptedException interrupted) {
Thread.currentThread().interrupt();
cancelled.set(true);
boolean abandoned = state.compareAndSet(0, 3);
if (abandoned) MAIN.removeCallbacks(work);
return failure("CANCELLED", "The caller stopped waiting",
mutation && !abandoned ? "unknown" : "not_started");
}
cancelled.set(true);
boolean abandoned = state.compareAndSet(0, 3);
if (abandoned) MAIN.removeCallbacks(work);
// A completion racing the deadline is still a usable receipt.
if (state.get() == 2 && result.get() != null) return result.get();
return failure("TIMEOUT", "The service main thread did not finish in time",
mutation && !abandoned ? "unknown" : "not_started");
}
private static JSONObject execute(
AccessibilityService service, JSONObject args, boolean mutation, AtomicBoolean cancelled) {
Resolution resolved = null;
Resolution scope = null;
JSONObject before = null;
boolean actionAttempted = false;
try {
Selector selector = new Selector(args.optJSONObject("selector"));
if (args.has("scope")) {
if (mutation) throw new IllegalArgumentException("scope is supported only for inspection");
Selector scopeSelector = new Selector(args.optJSONObject("scope"));
scope = resolve(service, scopeSelector, cancelled);
if (scope.error != null) return resolutionFailure(scope);
if (!scope.node.refresh() || !scopeSelector.matches(scope.node)) {
return failure("STALE_TARGET", "The scope changed before inspection", "not_started");
}
resolved = resolveCopy(scope.node, selector, cancelled);
} else {
resolved = resolve(service, selector, cancelled);
}
if (resolved.error != null) {
return resolutionFailure(resolved);
}
AccessibilityNodeInfo node = resolved.node;
if (!node.refresh() || !selector.matches(node)) {
return failure("STALE_TARGET", "The resolved node changed before use", "not_started");
}
if (scope != null && !withinScope(node, scope.node)) {
return failure("STALE_TARGET", "The target left the requested scope", "not_started");
}
before = snapshot(node);
if (!mutation) {
return new JSONObject().put("success", true).put("execution", "observed")
.put("code", "OK").put("node", before);
}
if (args.has("expected_token")) {
Object expected = args.get("expected_token");
if (!(expected instanceof String) || ((String) expected).isEmpty()) {
return withNodes(failure("INVALID_ARGUMENT", "expected_token must be a string",
"not_started"), before, null);
}
if (!MessageDigest.isEqual(((String) expected).getBytes(StandardCharsets.UTF_8),
before.getString("token").getBytes(StandardCharsets.UTF_8))) {
return withNodes(failure("STALE_TARGET", "The node snapshot token changed",
"not_started"), before, null);
}
}
Object rawAction = args.opt("action");
if (!(rawAction instanceof String)) {
return withNodes(failure("INVALID_ARGUMENT", "action must be a string",
"not_started"), before, null);
}
String action = (String) rawAction;
boolean replacement = "replace_selection".equals(action);
int id = actionId(replacement ? "set_text" : action);
if (id == 0 || !supports(node, id)
|| (replacement && (!supports(node, actionId("set_selection"))
|| !node.isEditable() || node.isPassword()))) {
return withNodes(failure("UNSUPPORTED", "The node does not support this action",
"not_started"), before, null);
}
if (!node.isEnabled()) {
return withNodes(failure("DISABLED", "The node is disabled", "not_started"), before, null);
}
Bundle parameters = new Bundle();
int caret = -1;
String expectedEditedText = null;
if ("set_progress".equals(action)) {
Object value = args.opt("value");
AccessibilityNodeInfo.RangeInfo range = node.getRangeInfo();
if (!(value instanceof Number) || range == null) {
return withNodes(failure("INVALID_ARGUMENT", "A numeric value and native range are required",
"not_started"), before, null);
}
double progress = ((Number) value).doubleValue();
float nativeProgress = (float) progress;
if (!Double.isFinite(progress) || !Float.isFinite(nativeProgress)
|| !Float.isFinite(range.getMin()) || !Float.isFinite(range.getMax())
|| progress < range.getMin() || progress > range.getMax()
|| (range.getType() == AccessibilityNodeInfo.RangeInfo.RANGE_TYPE_INT
&& progress != Math.rint(progress))) {
return withNodes(failure("INVALID_ARGUMENT", "value must be finite, in range, and integral for an integer range",
"not_started"), before, null);
}
parameters.putFloat(AccessibilityNodeInfo.ACTION_ARGUMENT_PROGRESS_VALUE, nativeProgress);
} else if ("set_selection".equals(action)) {
String text = nativeText(node);
int start = integer(args, "selection_start");
int end = integer(args, "selection_end");
validateSelection(text, start, end);
parameters.putInt(AccessibilityNodeInfo.ACTION_ARGUMENT_SELECTION_START_INT, start);
parameters.putInt(AccessibilityNodeInfo.ACTION_ARGUMENT_SELECTION_END_INT, end);
} else if ("set_text".equals(action) || replacement) {
Object value = args.opt("text");
if (!(value instanceof String)) {
return withNodes(failure("INVALID_ARGUMENT", "text must be a string",
"not_started"), before, null);
}
String text = (String) value;
if (text.length() > 65536) {
throw new IllegalArgumentException("text exceeds 65536 UTF-16 code units");
}
if (replacement) {
String original = nativeText(node);
boolean explicitStart = args.has("selection_start");
boolean explicitEnd = args.has("selection_end");
if (explicitStart != explicitEnd) {
throw new IllegalArgumentException("Provide both selection_start and selection_end");
}
int start = explicitStart ? integer(args, "selection_start") : node.getTextSelectionStart();
int end = explicitEnd ? integer(args, "selection_end") : node.getTextSelectionEnd();
validateSelection(original, start, end);
caret = start + text.length();
text = original.substring(0, start) + text + original.substring(end);
expectedEditedText = text;
}
parameters.putCharSequence(AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, text);
}
if (cancelled.get()) {
return withNodes(failure("CANCELLED", "The caller stopped waiting", "not_started"), before, null);
}
ControlWindowRoots.Blocked blocked = ControlWindowRoots.mutationBlock(service, node);
if (blocked != null) {
return withNodes(failure(blocked.code, blocked.message, "not_started"), before, null);
}
if (cancelled.get()) {
return withNodes(failure("CANCELLED", "The caller stopped waiting", "not_started"), before, null);
}
actionAttempted = true;
boolean accepted = node.performAction(id, parameters);
if (!accepted) {
return withNodes(failure("ACTION_REJECTED", "The native node rejected the action",
"not_started"), before, refreshedSnapshot(node));
}
if (replacement) {
if (cancelled.get()) {
return withNodes(failure("CANCELLED", "Text was accepted before the caller stopped waiting",
"unknown"), before, null);
}
// ACTION_SET_TEXT can invalidate the node; do not send a caret action to stale data.
if (!node.refresh()) {
return withNodes(failure("POST_EDIT_UNAVAILABLE", "Text was accepted but the caret target is unavailable",
"unknown"), before, null);
}
String edited = nativeText(node);
if (!edited.equals(expectedEditedText) || caret > edited.length() || splitsSurrogate(edited, caret)) {
return withNodes(failure("PARTIAL_EDIT", "Text was accepted but its resulting caret boundary changed",
"unknown"), before, snapshot(node));
}
Bundle selection = new Bundle();
selection.putInt(AccessibilityNodeInfo.ACTION_ARGUMENT_SELECTION_START_INT, caret);
selection.putInt(AccessibilityNodeInfo.ACTION_ARGUMENT_SELECTION_END_INT, caret);
if (cancelled.get()) {
return withNodes(failure("CANCELLED", "Text was accepted before the caller stopped waiting",
"unknown"), before, null);
}
blocked = ControlWindowRoots.mutationBlock(service, node);
if (blocked != null) {
return withNodes(failure(blocked.code, blocked.message, "unknown"), before, refreshedSnapshot(node));
}
if (cancelled.get()) {
return withNodes(failure("CANCELLED", "Text was accepted before the caller stopped waiting",
"unknown"), before, null);
}
if (!node.performAction(actionId("set_selection"), selection)) {
return withNodes(failure("PARTIAL_EDIT", "Text was accepted but the caret action was rejected",
"unknown"), before, refreshedSnapshot(node));
}
}
return withNodes(new JSONObject().put("success", true).put("execution", "completed")
.put("code", "OK"), before, refreshedSnapshot(node));
} catch (IllegalArgumentException invalid) {
return withNodes(failure("INVALID_ARGUMENT", invalid.getMessage(),
actionAttempted ? "unknown" : "not_started"), before, null);
} catch (Throwable error) {
return withNodes(failure("NATIVE_ERROR", "The native node could not be read or operated",
actionAttempted ? "unknown" : "not_started"), before, null);
} finally {
if (resolved != null) recycle(resolved.node);
if (scope != null) recycle(scope.node);
}
}
private static JSONObject refreshedSnapshot(AccessibilityNodeInfo node) {
try {
return node.refresh() ? snapshot(node) : null;
} catch (Throwable ignored) {
return null;
}
}
private static JSONObject withNodes(JSONObject result, JSONObject before, JSONObject after) {
try {
result.put("node_before", before == null ? JSONObject.NULL : before);
result.put("node_after", after == null ? JSONObject.NULL : after);
} catch (Exception ignored) {}
return result;
}
private static JSONObject failure(String code, String message, String execution) {
JSONObject result = new JSONObject();
try {
result.put("success", false).put("execution", execution).put("code", code)
.put("error", message == null ? code : message);
} catch (Exception ignored) {}
return result;
}
private static Resolution resolve(AccessibilityService service, Selector selector, AtomicBoolean cancelled) {
Resolution result = new Resolution();
try (ControlWindowRoots roots = ControlWindowRoots.capture(service, cancelled)) {
if (roots.error != null) {
result.error = roots.error;
result.message = roots.message;
} else {
for (int index = 0; index < roots.size() && result.error == null; index++) {
// One Resolution shares node/link budgets and match counts across every root.
visit(roots.take(index), selector, result, 0, cancelled);
}
}
}
return finishResolution(result);
}
@SuppressWarnings("deprecation")
private static Resolution resolveCopy(AccessibilityNodeInfo root, Selector selector, AtomicBoolean cancelled) {
return resolveRoot(AccessibilityNodeInfo.obtain(root), selector, cancelled);
}
private static Resolution resolveRoot(AccessibilityNodeInfo root, Selector selector, AtomicBoolean cancelled) {
Resolution result = new Resolution();
visit(root, selector, result, 0, cancelled);
return finishResolution(result);
}
private static Resolution finishResolution(Resolution result) {
if (result.error != null) {
recycle(result.node);
result.node = null;
} else if (result.matches == 0) {
result.error = "NOT_FOUND";
result.message = "No eligible native-window node matches the exact selector";
} else if (result.matches > 1) {
recycle(result.node);
result.node = null;
result.error = "AMBIGUOUS";
result.message = "More than one eligible native-window node matches the exact selector";
}
return result;
}
private static JSONObject resolutionFailure(Resolution resolved) throws Exception {
JSONObject error = failure(resolved.error, resolved.message, "not_started");
if (resolved.matches > 1) {
error.put("candidates", resolved.candidates);
error.put("candidate_count", resolved.matches);
}
return error;
}
@SuppressWarnings("deprecation")
private static boolean withinScope(AccessibilityNodeInfo node, AccessibilityNodeInfo scope) {
AccessibilityNodeInfo current = AccessibilityNodeInfo.obtain(node);
try {
for (int depth = 0; current != null && depth <= MAX_DEPTH; depth++) {
if (current.equals(scope)) return true;
AccessibilityNodeInfo parent = current.getParent();
recycle(current);
current = parent;
}
return false;
} finally {
recycle(current);
}
}
@SuppressWarnings("deprecation")
private static void visit(AccessibilityNodeInfo node, Selector selector, Resolution result, int depth,
AtomicBoolean cancelled) {
if (node == null) return;
try {
if (cancelled.get()) {
result.error = "CANCELLED";
result.message = "The caller stopped waiting during resolution";
return;
}
if (depth > MAX_DEPTH || result.examined >= MAX_NODES) {
result.error = "LIMIT_EXCEEDED";
result.message = "The native-window trees exceed the bounded search limit";
return;
}
if (result.error != null) return;
result.examined++;
if (selector.matches(node)) {
result.matches++;
if (result.candidates.length() < MAX_CANDIDATES) result.candidates.put(snapshot(node));
if (result.matches == 1) result.node = AccessibilityNodeInfo.obtain(node);
}
int count = node.getChildCount();
for (int index = 0; index < count && result.error == null; index++) {
if (result.links >= MAX_NODES) {
result.error = "LIMIT_EXCEEDED";
result.message = "The native-window trees exceed the bounded search limit";
break;
}
result.links++;
AccessibilityNodeInfo child = node.getChild(index);
if (child == null) {
result.error = "READ_FAILED";
result.message = "A declared child could not be read; absence and uniqueness are unknown";
break;
}
visit(child, selector, result, depth + 1, cancelled);
}
} catch (Throwable error) {
result.error = "READ_FAILED";
result.message = "The native-window tree changed during resolution";
} finally {
recycle(node);
}
}
private static final class Resolution {
AccessibilityNodeInfo node;
final JSONArray candidates = new JSONArray();
int matches;
int examined;
int links;
String error;
String message;
}
private static final class Selector {
private static final String[] SEMANTIC = {
"resource_id", "text", "content_desc", "class_name", "package_name"
};
final String[] values = new String[SEMANTIC.length];
Rect bounds;
Selector(JSONObject source) throws Exception {
if (source == null) throw new IllegalArgumentException("selector must be an object");
boolean meaningful = false;
Iterator<String> keys = source.keys();
while (keys.hasNext()) {
String key = keys.next();
boolean known = "bounds".equals(key);
for (String name : SEMANTIC) known |= name.equals(key);
if (!known) throw new IllegalArgumentException("Unknown selector attribute: " + key);
}
for (int i = 0; i < SEMANTIC.length; i++) {
if (!source.has(SEMANTIC[i])) continue;
Object value = source.get(SEMANTIC[i]);
if (!(value instanceof String)) {
throw new IllegalArgumentException("Selector semantic attributes must be strings");
}
values[i] = (String) value;
meaningful |= !values[i].trim().isEmpty();
}
if (!meaningful) throw new IllegalArgumentException("selector needs a nonempty semantic attribute");
if (source.has("bounds")) {
JSONArray array = source.optJSONArray("bounds");
if (array == null || array.length() != 4) {
throw new IllegalArgumentException("bounds must contain four integers");
}
int left = integer(array.get(0), "bounds");
int top = integer(array.get(1), "bounds");
int right = integer(array.get(2), "bounds");
int bottom = integer(array.get(3), "bounds");
if (right < left || bottom < top) throw new IllegalArgumentException("bounds are inverted");
bounds = new Rect(left, top, right, bottom);
}
}
boolean matches(AccessibilityNodeInfo node) {
String[] actual = {plain(node.getViewIdResourceName()), nativeText(node),
plain(node.getContentDescription()), plain(node.getClassName()), plain(node.getPackageName())};
for (int i = 0; i < values.length; i++) {
if (values[i] != null && !values[i].equals(actual[i])) return false;
}
if (bounds != null) {
Rect actualBounds = new Rect();
node.getBoundsInScreen(actualBounds);
if (!bounds.equals(actualBounds)) return false;
}
return true;
}
}
private static int integer(JSONObject object, String key) throws Exception {
return integer(object.opt(key), key);
}
private static int integer(Object raw, String key) {
if (!(raw instanceof Number)) throw new IllegalArgumentException(key + " must be an integer");
double value = ((Number) raw).doubleValue();
if (!Double.isFinite(value) || value != Math.rint(value)
|| value < Integer.MIN_VALUE || value > Integer.MAX_VALUE) {
throw new IllegalArgumentException(key + " must be an integer");
}
return (int) value;
}
private static void validateSelection(String text, int start, int end) {
if (start < 0 || end < start || end > text.length()
|| splitsSurrogate(text, start) || splitsSurrogate(text, end)) {
throw new IllegalArgumentException("Selection must be ordered UTF-16 boundaries within the text");
}
}
private static boolean splitsSurrogate(String text, int boundary) {
return boundary > 0 && boundary < text.length()
&& Character.isHighSurrogate(text.charAt(boundary - 1))
&& Character.isLowSurrogate(text.charAt(boundary));
}
private static String plain(CharSequence value) {
return value == null ? "" : value.toString();
}
private static String nativeText(AccessibilityNodeInfo node) {
if (Build.VERSION.SDK_INT >= 26 && node.isShowingHintText()) return "";
return plain(node.getText());
}
private static int actionId(String name) {
switch (name) {
case "click": return AccessibilityNodeInfo.ACTION_CLICK;
case "long_click": return AccessibilityNodeInfo.ACTION_LONG_CLICK;
case "focus": return AccessibilityNodeInfo.ACTION_FOCUS;
case "show_on_screen": return AccessibilityNodeInfo.AccessibilityAction.ACTION_SHOW_ON_SCREEN.getId();
case "set_progress": return AccessibilityNodeInfo.AccessibilityAction.ACTION_SET_PROGRESS.getId();
case "scroll_forward": return AccessibilityNodeInfo.ACTION_SCROLL_FORWARD;
case "scroll_backward": return AccessibilityNodeInfo.ACTION_SCROLL_BACKWARD;
case "scroll_up": return AccessibilityNodeInfo.AccessibilityAction.ACTION_SCROLL_UP.getId();
case "scroll_down": return AccessibilityNodeInfo.AccessibilityAction.ACTION_SCROLL_DOWN.getId();
case "scroll_left": return AccessibilityNodeInfo.AccessibilityAction.ACTION_SCROLL_LEFT.getId();
case "scroll_right": return AccessibilityNodeInfo.AccessibilityAction.ACTION_SCROLL_RIGHT.getId();
case "set_selection": return AccessibilityNodeInfo.ACTION_SET_SELECTION;
case "set_text": return AccessibilityNodeInfo.ACTION_SET_TEXT;
case "ime_enter": return Build.VERSION.SDK_INT >= 30
? AccessibilityNodeInfo.AccessibilityAction.ACTION_IME_ENTER.getId() : 0;
default: return 0;
}
}
private static boolean supports(AccessibilityNodeInfo node, int id) {
if (id == 0) return false;
List<AccessibilityNodeInfo.AccessibilityAction> actions = node.getActionList();
if (actions == null) return false;
for (AccessibilityNodeInfo.AccessibilityAction action : actions) {
if (action.getId() == id) return true;
}
return false;
}
/** Cheap native semantics only; the caller retains ownership of the live node. */
public static JSONObject metadata(AccessibilityNodeInfo node) {
JSONObject result = new JSONObject();
try {
result.put("range", JSONObject.NULL).put("actions", new JSONArray())
.put("selection_start", -1).put("selection_end", -1)
.put("editable", false).put("password", true);
if (node == null) return result;
boolean password = node.isPassword();
boolean editable = node.isEditable();
result.put("password", password).put("editable", editable);
result.put("selection_start", node.getTextSelectionStart());
result.put("selection_end", node.getTextSelectionEnd());
AccessibilityNodeInfo.RangeInfo range = node.getRangeInfo();
if (range != null) {
result.put("range", new JSONObject().put("min", range.getMin()).put("max", range.getMax())
.put("current", range.getCurrent()).put("type", range.getType()));
}
JSONArray actions = new JSONArray();
for (String action : ACTION_NAMES) {
if (supports(node, actionId(action))) actions.put(action);
}
if (!password && editable && supports(node, actionId("set_text"))
&& supports(node, actionId("set_selection"))) actions.put("replace_selection");
result.put("actions", actions);
} catch (Throwable ignored) {
// Preserve only fields actually read; do not manufacture supported actions.
}
return result;
}
private static JSONObject snapshot(AccessibilityNodeInfo node) throws Exception {
JSONObject result = metadata(node);
boolean password = result.optBoolean("password", true);
Rect bounds = new Rect();
node.getBoundsInScreen(bounds);
result.put("resource_id", plain(node.getViewIdResourceName()));
result.put("text", password ? "" : nativeText(node));
result.put("content_desc", plain(node.getContentDescription()));
result.put("class_name", plain(node.getClassName()));
result.put("package_name", plain(node.getPackageName()));
result.put("bounds", new JSONArray().put(bounds.left).put(bounds.top).put(bounds.right).put(bounds.bottom));
result.put("visible", node.isVisibleToUser());
result.put("enabled", node.isEnabled());
result.put("checked", node.isChecked());
result.put("checkable", node.isCheckable());
result.put("selected", node.isSelected());
result.put("focused", node.isFocused());
result.put("child_count", node.getChildCount());
result.put("scrollable", node.isScrollable());
ContentIdentity contents = new ContentIdentity();
appendContents(node, contents, 0);
result.put("content_signature", sha256(contents.identity.toString()));
AccessibilityNodeInfo.RangeInfo range = node.getRangeInfo();
// Explicit order and length prefixes keep the token independent of JSONObject map ordering.
StringBuilder identity = new StringBuilder();
String[] fields = {"resource_id", "text", "content_desc", "class_name", "package_name", "bounds",
"visible", "enabled", "editable", "checked", "checkable", "selected", "focused",
"password", "child_count", "scrollable", "selection_start", "selection_end", "actions"};
for (String field : fields) appendIdentity(identity, result.get(field).toString());
appendIdentity(identity, Integer.toString(node.getWindowId()));
if (range != null) {
appendIdentity(identity, Float.toString(range.getMin()));
appendIdentity(identity, Float.toString(range.getMax()));
appendIdentity(identity, Float.toString(range.getCurrent()));
appendIdentity(identity, Integer.toString(range.getType()));
} else {
appendIdentity(identity, "no-range");
}
result.put("token", sha256(identity.toString()));
return result;
}
private static final class ContentIdentity {
final StringBuilder identity = new StringBuilder();
int examined;
}
/** Bound content work independently of whole-window resolution; never retain live descendants. */
private static void appendContents(AccessibilityNodeInfo node, ContentIdentity contents, int depth) {
if (contents.examined >= 256 || depth > 8) {
appendIdentity(contents.identity, "bounded");
return;
}
contents.examined++;
if (!node.isVisibleToUser()) return;
Rect bounds = new Rect();
node.getBoundsInScreen(bounds);
appendIdentity(contents.identity, Integer.toString(depth));
appendIdentity(contents.identity, plain(node.getViewIdResourceName()));
appendIdentity(contents.identity, plain(node.getClassName()));
appendIdentity(contents.identity, node.isPassword() ? "" : nativeText(node));
appendIdentity(contents.identity, plain(node.getContentDescription()));
appendIdentity(contents.identity, bounds.flattenToString());
if (depth >= 8) {
appendIdentity(contents.identity, "depth-bound");
return;
}
int count = node.getChildCount();
for (int index = 0; index < count && contents.examined < 256; index++) {
AccessibilityNodeInfo child = node.getChild(index);
if (child == null) {
contents.examined++;
continue;
}
try {
appendContents(child, contents, depth + 1);
} finally {
recycle(child);
}
}
}
private static String sha256(String value) throws Exception {
byte[] digest = MessageDigest.getInstance("SHA-256")
.digest(value.getBytes(StandardCharsets.UTF_8));
char[] hex = "0123456789abcdef".toCharArray();
StringBuilder token = new StringBuilder(digest.length * 2);
for (byte octet : digest) {
token.append(hex[(octet & 255) >>> 4]).append(hex[octet & 15]);
}
return token.toString();
}
private static void appendIdentity(StringBuilder target, String value) {
target.append(value.length()).append(':').append(value);
}
@SuppressWarnings("deprecation")
private static void recycle(AccessibilityNodeInfo node) {
if (node != null) {
try { node.recycle(); } catch (Throwable ignored) {}
}
}
}
@@ -0,0 +1,630 @@
package com.artemis.helper;
import android.accessibilityservice.AccessibilityService;
import android.accessibilityservice.GestureDescription;
import android.graphics.Path;
import android.os.Build;
import android.os.Handler;
import android.os.Looper;
import android.os.SystemClock;
import org.json.JSONArray;
import org.json.JSONObject;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import java.util.regex.Pattern;
/** One bounded, callback-driven continuous gesture; ambiguous input is never replayed. */
public final class TouchGestures {
private static final Handler MAIN = new Handler(Looper.getMainLooper());
private static final AtomicReference<Object> ACTIVE = new AtomicReference<>();
private static final Object REGISTRATION = new Object();
private static final int MAX_CANCELLATIONS = 64;
private static final long CANCELLATION_TTL_MS = 60000L;
private static final LinkedHashMap<String, Long> CANCELLED = new LinkedHashMap<>();
// An evicted live tombstone must not make its delayed request runnable.
private static long cancellationBarrierUntil;
private static final Pattern OPERATION_ID = Pattern.compile("[A-Za-z0-9_-]{1,80}");
private TouchGestures() {}
private static void pruneCancellations(long now) {
Iterator<Map.Entry<String, Long>> entries = CANCELLED.entrySet().iterator();
while (entries.hasNext()) {
if (entries.next().getValue() <= now) entries.remove();
}
}
private static void rememberCancellation(String operationId, long now) {
pruneCancellations(now);
CANCELLED.remove(operationId);
CANCELLED.put(operationId, now + CANCELLATION_TTL_MS);
if (CANCELLED.size() > MAX_CANCELLATIONS) {
Iterator<Map.Entry<String, Long>> entries = CANCELLED.entrySet().iterator();
Map.Entry<String, Long> oldest = entries.next();
cancellationBarrierUntil = Math.max(cancellationBarrierUntil, oldest.getValue());
entries.remove();
}
}
/** Legacy and continuous gestures share one channel, including unknown input. */
static boolean claimLegacy(Object owner) {
synchronized (REGISTRATION) {
long now = SystemClock.uptimeMillis();
pruneCancellations(now);
return now >= cancellationBarrierUntil && ACTIVE.compareAndSet(null, owner);
}
}
static void releaseLegacy(Object owner) {
// Releasing does not acquire REGISTRATION: Runner's monitor may be held,
// while cancellation takes REGISTRATION before that same monitor.
ACTIVE.compareAndSet(owner, null);
}
public static JSONObject perform(AccessibilityService service, JSONObject args) {
String operationId = safeOperationId(args);
long enteredAt = SystemClock.uptimeMillis();
if (service == null || args == null) {
return receipt(false, "not_started", "INVALID_ARGUMENT", operationId, 0, 0, 0, enteredAt,
"A service and arguments are required");
}
if (Thread.currentThread().isInterrupted()) {
return receipt(false, "not_started", "CANCELLED", operationId, 0, 0, 0, enteredAt,
"The caller was interrupted before gesture registration");
}
if (Build.VERSION.SDK_INT < 26) {
return receipt(false, "not_started", "UNSUPPORTED", operationId, 0, 0, 0, enteredAt,
"Continuous gestures require Android 8.0 / API 26 or newer");
}
if (Looper.myLooper() == Looper.getMainLooper()) {
return receipt(false, "not_started", "INVALID_CONTEXT", operationId, 0, 0, 0, enteredAt,
"Gesture waiting must run outside the service main thread");
}
final Spec spec;
try {
spec = new Spec(args);
DisplayUtils.DisplayInfo display = DisplayUtils.getDisplayInfo(service);
if (display.width != spec.width || display.height != spec.height) {
return receipt(false, "not_started", "DISPLAY_CHANGED", spec.operationId, 0, 0,
spec.fingers, enteredAt, "screen_size differs from the current physical display");
}
} catch (IllegalArgumentException invalid) {
return receipt(false, "not_started", "INVALID_ARGUMENT", operationId, 0, 0, 0, enteredAt,
invalid.getMessage());
} catch (Throwable invalid) {
return receipt(false, "not_started", "INVALID_ARGUMENT", operationId, 0, 0, 0, enteredAt,
"The gesture arguments could not be read");
}
Runner runner = new Runner(service, spec);
synchronized (REGISTRATION) {
long now = SystemClock.uptimeMillis();
pruneCancellations(now);
if (CANCELLED.containsKey(spec.operationId)) {
return receipt(false, "not_started", "CANCELLED", spec.operationId, 0, 0,
spec.fingers, enteredAt, "Cancellation was recorded before this gesture registered");
}
if (now < cancellationBarrierUntil || !ACTIVE.compareAndSet(null, runner)) {
return receipt(false, "not_started", "BUSY", spec.operationId, 0, 0, spec.fingers,
enteredAt, "The native input channel is owned or pending cancellation expiry");
}
}
if (!MAIN.post(runner.start)) {
ACTIVE.compareAndSet(runner, null);
return receipt(false, "not_started", "UNAVAILABLE", spec.operationId, 0, 0, spec.fingers,
enteredAt, "The service main thread is unavailable");
}
try {
long remaining = Math.max(1L, runner.deadlineAt + 500L - SystemClock.uptimeMillis());
if (runner.done.await(remaining, TimeUnit.MILLISECONDS)) return runner.reply.get();
runner.abandon("TIMEOUT", "The gesture did not finish within its bounded deadline");
} catch (InterruptedException interrupted) {
Thread.currentThread().interrupt();
runner.abandon("CANCELLED", "The caller stopped waiting for the gesture");
}
return runner.reply.get();
}
/** Records cancellation before registration too; active completion belongs to perform. */
public static JSONObject cancel(JSONObject args) {
long enteredAt = SystemClock.uptimeMillis();
String operationId = safeOperationId(args);
if (operationId.isEmpty()) {
return receipt(false, "not_started", "INVALID_ARGUMENT", "", 0, 0, 0, enteredAt,
"operation_id must match [A-Za-z0-9_-]{1,80}");
}
final Runner runner;
synchronized (REGISTRATION) {
rememberCancellation(operationId, enteredAt);
Object owner = ACTIVE.get();
if (!(owner instanceof Runner) || !operationId.equals(((Runner) owner).spec.operationId)) {
return receipt(true, "observed", "CANCEL_RECORDED", operationId, 0, 0, 0,
enteredAt, "Cancellation recorded; no matching active gesture was found");
}
runner = (Runner) owner;
synchronized (runner) {
if (runner.ended || ACTIVE.get() != runner) {
return receipt(true, "observed", "CANCEL_RECORDED", operationId, 0, 0,
runner.spec.fingers, enteredAt, "Cancellation recorded after the gesture ended");
}
runner.cancelRequested = true;
}
}
MAIN.post(runner.cancelOnMain);
return runner.snapshot(false, "unknown", "CANCEL_REQUESTED",
"Cancellation was requested; await the original gesture receipt");
}
private static String safeOperationId(JSONObject args) {
if (args == null) return "";
Object value = args.opt("operation_id");
return value instanceof String && OPERATION_ID.matcher((String) value).matches() ? (String) value : "";
}
private static int integer(Object value, String name) {
if (!(value instanceof Number)) throw new IllegalArgumentException(name + " must be an integer");
double number = ((Number) value).doubleValue();
if (!Double.isFinite(number) || number != Math.rint(number)
|| number < Integer.MIN_VALUE || number > Integer.MAX_VALUE) {
throw new IllegalArgumentException(name + " must be an integer");
}
return (int) number;
}
private static final class Spec {
final String operationId;
final String requireVisible;
final int width;
final int height;
final int fingers;
final int[] times;
final int[][][] points;
final int duration;
Spec(JSONObject args) throws Exception {
operationId = safeOperationId(args);
if (operationId.isEmpty()) {
throw new IllegalArgumentException("operation_id must match [A-Za-z0-9_-]{1,80}");
}
if (args.has("require_visible")) {
Object guard = args.get("require_visible");
if (!(guard instanceof String) || ((String) guard).trim().isEmpty()) {
throw new IllegalArgumentException("require_visible must be a nonempty exact resource ID");
}
requireVisible = (String) guard;
} else {
requireVisible = null;
}
JSONArray size = args.optJSONArray("screen_size");
if (size == null || size.length() != 2) {
throw new IllegalArgumentException("screen_size must contain physical width and height");
}
width = integer(size.get(0), "screen_size");
height = integer(size.get(1), "screen_size");
if (width <= 0 || height <= 0) throw new IllegalArgumentException("screen_size must be positive");
JSONArray frames = args.optJSONArray("frames");
if (frames == null || frames.length() < 2 || frames.length() > 32) {
throw new IllegalArgumentException("frames must contain 2 through 32 keyframes");
}
times = new int[frames.length()];
points = new int[frames.length()][][];
int pointerCount = 0;
for (int frameIndex = 0; frameIndex < frames.length(); frameIndex++) {
JSONObject frame = frames.optJSONObject(frameIndex);
if (frame == null) throw new IllegalArgumentException("Each frame must be an object");
int at = integer(frame.opt("at_ms"), "at_ms");
if ((frameIndex == 0 && at != 0) || at < 0 || at > 10000
|| (frameIndex > 0 && at <= times[frameIndex - 1])) {
throw new IllegalArgumentException("Frame times must strictly increase from 0 through at most 10000 ms");
}
times[frameIndex] = at;
JSONArray positions = frame.optJSONArray("points");
if (positions == null || positions.length() < 1 || positions.length() > 5
|| (frameIndex > 0 && positions.length() != pointerCount)) {
throw new IllegalArgumentException("Every frame must contain the same 1 through 5 fingers");
}
pointerCount = positions.length();
points[frameIndex] = new int[pointerCount][2];
for (int finger = 0; finger < pointerCount; finger++) {
JSONArray point = positions.optJSONArray(finger);
if (point == null || point.length() != 2) {
throw new IllegalArgumentException("Each finger position must contain x and y");
}
int x = integer(point.get(0), "point x");
int y = integer(point.get(1), "point y");
if (x < 0 || y < 0 || x >= width || y >= height) {
throw new IllegalArgumentException("Every point must lie within the physical screen bounds");
}
points[frameIndex][finger][0] = x;
points[frameIndex][finger][1] = y;
}
}
fingers = pointerCount;
duration = times[times.length - 1];
}
}
private static final class Attempt {
final GestureDescription.StrokeDescription[] strokes;
final int endFrame;
final boolean continuing;
final boolean release;
boolean counted;
long earliestEndAt;
Attempt(GestureDescription.StrokeDescription[] strokes, int endFrame,
boolean continuing, boolean release) {
this.strokes = strokes;
this.endFrame = endFrame;
this.continuing = continuing;
this.release = release;
}
}
private static final class Runner {
final AccessibilityService service;
final Spec spec;
final long startedAt = SystemClock.uptimeMillis();
final long deadlineAt;
final CountDownLatch done = new CountDownLatch(1);
final AtomicReference<JSONObject> reply = new AtomicReference<>();
volatile int completedSegments;
volatile int dispatchedSegments;
volatile boolean ended;
boolean cancelRequested; // Read/written under this runner's monitor.
boolean everAttempted;
boolean uncertainInput;
boolean releaseAttempted;
Attempt current;
GestureDescription.StrokeDescription[] held;
int heldFrame;
String stopCode;
String stopError;
Runner(AccessibilityService service, Spec spec) {
this.service = service;
this.spec = spec;
deadlineAt = startedAt + spec.duration + 4500L;
}
final Runnable start = new Runnable() {
@Override public void run() {
if (ended) return;
if (SystemClock.uptimeMillis() >= deadlineAt) {
finish(false, "not_started", "TIMEOUT", "The queued gesture expired before input", true);
return;
}
MAIN.postDelayed(deadline, Math.max(1L, deadlineAt - SystemClock.uptimeMillis()));
if (isCancelled()) {
finish(false, "not_started", "CANCELLED", "Cancelled before input started", true);
return;
}
try {
if (spec.requireVisible != null) {
JSONObject guard = NativeControls.inspect(service, new JSONObject().put("selector",
new JSONObject().put("resource_id", spec.requireVisible)));
if (!guard.optBoolean("success", false)) {
finish(false, "not_started", guard.optString("code", "NOT_FOUND"),
"The required native target could not be uniquely revalidated", true);
return;
}
JSONObject node = guard.optJSONObject("node");
if (node == null || !node.optBoolean("visible") || !node.optBoolean("enabled")) {
finish(false, "not_started", "NOT_FOUND", "The required native target is not visible and enabled", true);
return;
}
}
if (!displayMatches()) {
finish(false, "not_started", "DISPLAY_CHANGED", "The physical display changed before input", true);
return;
}
dispatchSegment(0);
} catch (Throwable failure) {
stop("NATIVE_ERROR", "Native validation or gesture construction failed", true);
}
}
};
final Runnable cancelOnMain = new Runnable() {
@Override public void run() {
if (ended) return;
synchronized (Runner.this) {
if (!everAttempted) {
finish(false, "not_started", "CANCELLED", "Cancelled before input started", true);
return;
}
}
// An in-flight interval finishes first; its callback owns the endpoint release.
if (current == null && held != null && !releaseAttempted) release();
}
};
final Runnable deadline = new Runnable() {
@Override public void run() {
if (ended) return;
synchronized (Runner.this) { cancelRequested = true; }
stopCode = "TIMEOUT";
stopError = "The native gesture exceeded its bounded deadline";
if (current == null && held != null && !releaseAttempted) release();
if (!ended) {
boolean attempted;
synchronized (Runner.this) { attempted = everAttempted; }
publish(false, attempted ? "unknown" : "not_started", "TIMEOUT", stopError);
if (!attempted) quietEnd();
}
}
};
private boolean isCancelled() {
synchronized (this) { return cancelRequested; }
}
private boolean displayMatches() {
DisplayUtils.DisplayInfo display = DisplayUtils.getDisplayInfo(service);
return display.width == spec.width && display.height == spec.height;
}
private void dispatchSegment(int fromFrame) {
if (ended) return;
if (isCancelled()) {
if (held != null) release();
else finish(false, everAttempted ? "cancelled" : "not_started", "CANCELLED",
"Cancelled before the next movement interval", true);
return;
}
if (SystemClock.uptimeMillis() >= deadlineAt) {
stop("TIMEOUT", "The gesture deadline expired before the next interval", false);
return;
}
if (!displayMatches()) {
stop("DISPLAY_CHANGED", "The physical display changed during the gesture", false);
return;
}
int endFrame = fromFrame + 1;
boolean continuing = endFrame < spec.times.length - 1;
int interval = spec.times[endFrame] - spec.times[fromFrame];
GestureDescription.StrokeDescription[] strokes = new GestureDescription.StrokeDescription[spec.fingers];
try {
for (int finger = 0; finger < spec.fingers; finger++) {
Path path = path(spec.points[fromFrame][finger], spec.points[endFrame][finger]);
strokes[finger] = held == null
? new GestureDescription.StrokeDescription(path, 0L, interval, continuing)
: held[finger].continueStroke(path, 0L, interval, continuing);
}
} catch (Throwable invalid) {
stop("NATIVE_ERROR", "A continuous gesture interval could not be constructed", false);
return;
}
dispatch(new Attempt(strokes, endFrame, continuing, false));
}
private void release() {
if (ended || releaseAttempted || held == null) return;
releaseAttempted = true;
try {
GestureDescription.StrokeDescription[] strokes = new GestureDescription.StrokeDescription[spec.fingers];
for (int finger = 0; finger < spec.fingers; finger++) {
int[] endpoint = spec.points[heldFrame][finger];
strokes[finger] = held[finger].continueStroke(path(endpoint, endpoint), 0L, 1L, false);
}
dispatch(new Attempt(strokes, heldFrame, false, true));
} catch (Throwable invalid) {
uncertainInput = true;
finish(false, "unknown", "RELEASE_ERROR", "The continuing pointers could not be released", false);
}
}
private void dispatch(final Attempt attempt) {
final GestureDescription gesture;
try {
GestureDescription.Builder builder = new GestureDescription.Builder();
for (GestureDescription.StrokeDescription stroke : attempt.strokes) builder.addStroke(stroke);
gesture = builder.build();
} catch (Throwable invalid) {
if (attempt.release) {
uncertainInput = true;
finish(false, "unknown", "RELEASE_ERROR", "The endpoint release could not be constructed", false);
} else {
stop("NATIVE_ERROR", "The gesture interval could not be constructed", false);
}
return;
}
if (!attempt.release) {
synchronized (this) {
if (cancelRequested) {
if (held != null) release();
else finish(false, everAttempted ? "cancelled" : "not_started", "CANCELLED",
"Cancelled before movement dispatch", true);
return;
}
if (SystemClock.uptimeMillis() >= deadlineAt) {
stop("TIMEOUT", "The gesture deadline expired before dispatch", false);
return;
}
// This monitor closes cancellation-before-start versus dispatch races.
everAttempted = true;
}
}
current = attempt;
if (!attempt.release) {
attempt.earliestEndAt = SystemClock.uptimeMillis()
+ spec.times[attempt.endFrame] - spec.times[attempt.endFrame - 1];
}
try {
boolean accepted = service.dispatchGesture(gesture, new AccessibilityService.GestureResultCallback() {
@Override public void onCompleted(GestureDescription description) {
callback(attempt, true);
}
@Override public void onCancelled(GestureDescription description) {
callback(attempt, false);
}
}, MAIN);
if (!accepted) {
current = null;
if (attempt.release) {
uncertainInput = true;
finish(false, "unknown", "RELEASE_REJECTED", "The continuing-pointer release was rejected", false);
} else {
stop("DISPATCH_REJECTED", "Android rejected a gesture interval", false);
}
return;
}
if (!attempt.release) {
attempt.counted = true;
dispatchedSegments++;
held = null;
}
} catch (Throwable failure) {
uncertainInput = true;
stopCode = "DISPATCH_ERROR";
stopError = "Dispatch threw after input might have started";
if (attempt.release) {
finish(false, "unknown", "RELEASE_ERROR", "Release dispatch completion is unknown", false);
} else if (held != null) {
// Release once from the last confirmed continuing endpoint; never retry movement.
release();
} else {
// A late callback can still prove the endpoint and allow a safe release.
publish(false, "unknown", "DISPATCH_ERROR", stopError);
}
}
}
private void callback(Attempt attempt, boolean completed) {
if (ended || current != attempt) return;
current = null;
if (attempt.release) {
if (!completed) {
uncertainInput = true;
finish(false, "unknown", "RELEASE_CANCELLED", "The endpoint release did not complete", false);
return;
}
held = null;
finish(false, uncertainInput ? "unknown" : "cancelled",
stopCode == null ? "CANCELLED" : stopCode,
stopError == null ? "Cancelled after releasing the current endpoint" : stopError, true);
return;
}
if (!attempt.counted) {
attempt.counted = true;
dispatchedSegments++;
}
if (!completed) {
held = null;
finish(false, uncertainInput ? "unknown" : "cancelled", "GESTURE_CANCELLED",
"Android cancelled the in-flight gesture; partial input is possible", true);
return;
}
completedSegments++;
if (attempt.continuing) {
held = attempt.strokes;
heldFrame = attempt.endFrame;
} else {
held = null;
}
if (isCancelled() || stopCode != null) {
if (held != null) release();
else finish(false, uncertainInput ? "unknown" : "cancelled",
stopCode == null ? "CANCELLED" : stopCode,
stopError == null ? "Cancellation was requested before completion" : stopError, true);
} else if (attempt.endFrame == spec.times.length - 1) {
finish(true, "completed", "OK", null, true);
} else {
// Android can finish an intermediate stationary path as soon as
// its DOWN event is injected: no MOVE/UP is due at the requested
// end time. Preserve the continuing stroke and the declared hold
// before starting the next interval. Cancellation may release it.
long remaining = attempt.earliestEndAt - SystemClock.uptimeMillis();
if (remaining > 0) {
MAIN.postDelayed(new Runnable() {
@Override public void run() {
if (!ended) dispatchSegment(attempt.endFrame);
}
}, remaining);
} else {
dispatchSegment(attempt.endFrame);
}
}
}
private void stop(String code, String error, boolean definitelyBeforeInput) {
stopCode = code;
stopError = error;
if (held != null) {
release();
} else if (current != null) {
synchronized (this) { cancelRequested = true; }
publish(false, "unknown", code, error);
} else {
finish(false, definitelyBeforeInput || dispatchedSegments == 0 ? "not_started" : "cancelled",
code, error, true);
}
}
void abandon(final String code, final String error) {
final boolean attempted;
synchronized (this) {
if (ended) return;
cancelRequested = true;
attempted = everAttempted;
}
MAIN.post(cancelOnMain);
publish(false, attempted ? "unknown" : "not_started", code, error);
}
private void quietEnd() {
synchronized (this) {
if (ended) return;
ended = true;
}
MAIN.removeCallbacks(deadline);
ACTIVE.compareAndSet(this, null);
}
private void finish(boolean success, String execution, String code, String error, boolean quiet) {
if (quiet) {
synchronized (this) {
quietEnd();
publish(success, execution, code, error);
}
} else {
publish(success, execution, code, error);
}
}
private void publish(boolean success, String execution, String code, String error) {
JSONObject result = snapshot(success, execution, code, error);
if (reply.compareAndSet(null, result)) done.countDown();
}
JSONObject snapshot(boolean success, String execution, String code, String error) {
return receipt(success, execution, code, spec.operationId, completedSegments, dispatchedSegments,
spec.fingers, startedAt, error);
}
}
private static Path path(int[] start, int[] end) {
Path path = new Path();
path.moveTo(start[0], start[1]);
path.lineTo(end[0], end[1]);
return path;
}
private static JSONObject receipt(boolean success, String execution, String code, String operationId,
int completed, int dispatched, int fingers, long startedAt, String error) {
JSONObject result = new JSONObject();
try {
result.put("success", success).put("execution", execution).put("code", code)
.put("operation_id", operationId).put("completed_segments", completed)
.put("dispatched_segments", dispatched).put("fingers", fingers)
.put("elapsed_ms", Math.max(0L, SystemClock.uptimeMillis() - startedAt))
.put("error", error == null ? JSONObject.NULL : error);
} catch (Exception ignored) {}
return result;
}
}
@@ -1,7 +1,7 @@
{
"package": "com.artemis.helper",
"version_code": 9,
"version_name": "1.2.3",
"sha256": "f25f4b6b173049def6601953d8aa43779f3e5c5775d9a74f46dff7e93b731123",
"built_at": "2026-10-26T13:49:10+00:00"
"version_code": 15,
"version_name": "1.3.0",
"sha256": "544c38d5c135518fe6796aa4e03f804e87342687125a383ff3244952c8160d1f",
"built_at": "2026-11-02T13:16:01+00:00"
}