feat(android): add native controls and continuous multi-touch
This commit is contained in:
chenmingxuan committed
1 parent
3dd96078db
commit
4b12ae7728
70 files changed
+13105
-179
No files matched your search
File diff suppressed because it is too large.
Load diff
+1390
-1
File diff suppressed because it is too large.
Load diff
+1462
-73
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,296 @@
|
||||
package com.artemis.fixture;
|
||||
|
||||
import android.app.Activity;
|
||||
import android.graphics.Canvas;
|
||||
import android.graphics.Color;
|
||||
import android.graphics.Paint;
|
||||
import android.os.Bundle;
|
||||
import android.text.Editable;
|
||||
import android.text.InputType;
|
||||
import android.text.TextWatcher;
|
||||
import android.view.MotionEvent;
|
||||
import android.view.View;
|
||||
import android.view.accessibility.AccessibilityNodeInfo;
|
||||
import android.view.inputmethod.EditorInfo;
|
||||
import android.widget.Button;
|
||||
import android.widget.EditText;
|
||||
import android.widget.LinearLayout;
|
||||
import android.widget.ScrollView;
|
||||
import android.widget.SeekBar;
|
||||
import android.widget.TextView;
|
||||
|
||||
import org.json.JSONArray;
|
||||
import org.json.JSONObject;
|
||||
|
||||
import java.io.FileOutputStream;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
|
||||
/** Disposable, dependency-free native controls and observable touch-event fixture. */
|
||||
public final class MainActivity extends Activity {
|
||||
private TextView status;
|
||||
private TextView touchStatus;
|
||||
private EditText editor;
|
||||
private SeekBar range;
|
||||
private TouchSurface touch;
|
||||
private int clicks;
|
||||
private int longClicks;
|
||||
private int imeEnters;
|
||||
private int statusVersion;
|
||||
|
||||
@Override public void onCreate(Bundle savedInstanceState) {
|
||||
super.onCreate(savedInstanceState);
|
||||
setTitle("ARTEMIS native actions fixture");
|
||||
ScrollView scroll = new ScrollView(this);
|
||||
scroll.setContentDescription("fixture.scroll");
|
||||
scroll.setFillViewport(true);
|
||||
LinearLayout body = new LinearLayout(this);
|
||||
body.setOrientation(LinearLayout.VERTICAL);
|
||||
body.setPadding(dp(12), dp(12), dp(12), dp(24));
|
||||
scroll.addView(body);
|
||||
body.addView(label("ARTEMIS native controls", "fixture.title"));
|
||||
|
||||
status = label("status:ready", "fixture.status");
|
||||
body.addView(status);
|
||||
range = new SeekBar(this);
|
||||
range.setContentDescription("fixture.range");
|
||||
range.setMax(100);
|
||||
range.setProgress(25);
|
||||
body.addView(range, new LinearLayout.LayoutParams(-1, dp(52)));
|
||||
range.setOnSeekBarChangeListener(new SeekBar.OnSeekBarChangeListener() {
|
||||
@Override public void onProgressChanged(SeekBar seekBar, int progress, boolean fromUser) {
|
||||
updateStatus("range:" + progress);
|
||||
}
|
||||
@Override public void onStartTrackingTouch(SeekBar seekBar) {}
|
||||
@Override public void onStopTrackingTouch(SeekBar seekBar) { writeState(); }
|
||||
});
|
||||
|
||||
Button click = button("Native click", "fixture.click");
|
||||
click.setOnClickListener(new View.OnClickListener() {
|
||||
@Override public void onClick(View view) { clicks++; updateStatus("click:" + clicks); }
|
||||
});
|
||||
body.addView(click);
|
||||
Button longClick = button("Native long click", "fixture.long_click");
|
||||
longClick.setOnLongClickListener(new View.OnLongClickListener() {
|
||||
@Override public boolean onLongClick(View view) {
|
||||
longClicks++;
|
||||
updateStatus("long_click:" + longClicks);
|
||||
return true;
|
||||
}
|
||||
});
|
||||
body.addView(longClick);
|
||||
Button delayed = button("Change status after 700 ms", "fixture.delayed");
|
||||
delayed.setOnClickListener(new View.OnClickListener() {
|
||||
@Override public void onClick(View view) {
|
||||
updateStatus("status:waiting");
|
||||
status.postDelayed(new Runnable() {
|
||||
@Override public void run() { statusVersion++; updateStatus("status:changed:" + statusVersion); }
|
||||
}, 700L);
|
||||
}
|
||||
});
|
||||
body.addView(delayed);
|
||||
|
||||
editor = new EditText(this) {
|
||||
@Override protected void onSelectionChanged(int start, int end) {
|
||||
super.onSelectionChanged(start, end);
|
||||
writeState();
|
||||
}
|
||||
};
|
||||
editor.setContentDescription("fixture.editor");
|
||||
editor.setInputType(InputType.TYPE_CLASS_TEXT);
|
||||
editor.setSingleLine(true);
|
||||
editor.setImeOptions(EditorInfo.IME_ACTION_DONE);
|
||||
editor.setText("A\uD83D\uDE00B中文");
|
||||
editor.setSelection(editor.length());
|
||||
editor.setOnEditorActionListener(new TextView.OnEditorActionListener() {
|
||||
@Override public boolean onEditorAction(TextView view, int actionId, android.view.KeyEvent event) {
|
||||
imeEnters++;
|
||||
updateStatus("ime_enter:" + imeEnters);
|
||||
return true;
|
||||
}
|
||||
});
|
||||
editor.addTextChangedListener(new TextWatcher() {
|
||||
@Override public void beforeTextChanged(CharSequence value, int start, int count, int after) {}
|
||||
@Override public void onTextChanged(CharSequence value, int start, int before, int count) {}
|
||||
@Override public void afterTextChanged(Editable value) { writeState(); }
|
||||
});
|
||||
body.addView(editor, new LinearLayout.LayoutParams(-1, dp(52)));
|
||||
|
||||
touchStatus = label("touch:ready", "fixture.touch_status");
|
||||
body.addView(touchStatus);
|
||||
touch = new TouchSurface();
|
||||
body.addView(touch, new LinearLayout.LayoutParams(-1, dp(210)));
|
||||
Button reset = button("Reset touch statistics", "fixture.touch_reset");
|
||||
reset.setOnClickListener(new View.OnClickListener() {
|
||||
@Override public void onClick(View view) { touch.reset(); writeState(); }
|
||||
});
|
||||
body.addView(reset);
|
||||
// Native ScrollView actions must traverse multiple viewports to reach this target.
|
||||
for (int i = 0; i < 45; i++) {
|
||||
TextView row = label("Native list row " + i, "fixture.row." + i);
|
||||
row.setMinHeight(dp(64));
|
||||
body.addView(row);
|
||||
}
|
||||
Button bottom = button("Bottom target", "fixture.bottom");
|
||||
bottom.setOnClickListener(new View.OnClickListener() {
|
||||
@Override public void onClick(View view) { updateStatus("bottom:clicked"); }
|
||||
});
|
||||
body.addView(bottom);
|
||||
setContentView(scroll);
|
||||
writeState();
|
||||
}
|
||||
|
||||
private int dp(int value) { return Math.round(value * getResources().getDisplayMetrics().density); }
|
||||
|
||||
private TextView label(String text, String description) {
|
||||
TextView result = new TextView(this);
|
||||
result.setText(text);
|
||||
result.setTextSize(16);
|
||||
result.setTextColor(Color.BLACK);
|
||||
result.setPadding(0, dp(4), 0, dp(4));
|
||||
result.setContentDescription(description);
|
||||
return result;
|
||||
}
|
||||
|
||||
private Button button(String text, String description) {
|
||||
Button result = new Button(this);
|
||||
result.setText(text);
|
||||
result.setContentDescription(description);
|
||||
return result;
|
||||
}
|
||||
|
||||
private void updateStatus(String text) {
|
||||
status.setText(text);
|
||||
writeState();
|
||||
}
|
||||
|
||||
/** Read explicitly with: adb shell run-as com.artemis.fixture cat files/native_actions_state.json */
|
||||
private void writeState() {
|
||||
if (editor == null || range == null || touch == null) return;
|
||||
try {
|
||||
JSONObject state = new JSONObject().put("click_count", clicks).put("long_click_count", longClicks)
|
||||
.put("ime_enter_count", imeEnters).put("status", status.getText().toString())
|
||||
.put("progress", range.getProgress()).put("text", editor.getText().toString())
|
||||
.put("selection_start", editor.getSelectionStart()).put("selection_end", editor.getSelectionEnd())
|
||||
.put("touch", touch.state(true));
|
||||
try (FileOutputStream stream = openFileOutput("native_actions_state.json", MODE_PRIVATE)) {
|
||||
stream.write(state.toString().getBytes(StandardCharsets.UTF_8));
|
||||
}
|
||||
} catch (Exception ignored) {}
|
||||
}
|
||||
|
||||
private final class TouchSurface extends View {
|
||||
private final Paint paint = new Paint(Paint.ANTI_ALIAS_FLAG);
|
||||
private JSONArray events = new JSONArray();
|
||||
private int downs;
|
||||
private int ups;
|
||||
private int cancels;
|
||||
private int moves;
|
||||
private int maxPointers;
|
||||
private int pointerDowns;
|
||||
private int pointerUps;
|
||||
private double initialDistance;
|
||||
private double initialAngle;
|
||||
private double scale = 1.0;
|
||||
private double rotation;
|
||||
|
||||
TouchSurface() {
|
||||
super(MainActivity.this);
|
||||
setContentDescription("fixture.touch");
|
||||
setImportantForAccessibility(View.IMPORTANT_FOR_ACCESSIBILITY_YES);
|
||||
setFocusable(true);
|
||||
setClickable(true);
|
||||
setBackgroundColor(Color.rgb(208, 239, 247));
|
||||
paint.setColor(Color.rgb(15, 45, 60));
|
||||
paint.setTextSize(dp(16));
|
||||
}
|
||||
|
||||
void reset() {
|
||||
events = new JSONArray();
|
||||
downs = ups = cancels = moves = maxPointers = pointerDowns = pointerUps = 0;
|
||||
initialDistance = initialAngle = rotation = 0;
|
||||
scale = 1;
|
||||
publish();
|
||||
}
|
||||
|
||||
JSONObject state(boolean includeEvents) throws Exception {
|
||||
JSONObject result = new JSONObject().put("down_count", downs).put("up_count", ups)
|
||||
.put("cancel_count", cancels).put("move_count", moves).put("max_pointer_count", maxPointers)
|
||||
.put("pointer_down_count", pointerDowns).put("pointer_up_count", pointerUps)
|
||||
.put("scale", scale).put("rotation_degrees", rotation);
|
||||
if (includeEvents) result.put("events", events);
|
||||
return result;
|
||||
}
|
||||
|
||||
private void publish() {
|
||||
try { touchStatus.setText(state(false).toString()); } catch (Exception ignored) {}
|
||||
invalidate();
|
||||
writeState();
|
||||
}
|
||||
|
||||
@Override public boolean onTouchEvent(MotionEvent event) {
|
||||
int action = event.getActionMasked();
|
||||
if (action == MotionEvent.ACTION_DOWN) {
|
||||
downs++;
|
||||
initialDistance = 0;
|
||||
scale = 1;
|
||||
rotation = 0;
|
||||
getParent().requestDisallowInterceptTouchEvent(true);
|
||||
} else if (action == MotionEvent.ACTION_UP) {
|
||||
ups++;
|
||||
getParent().requestDisallowInterceptTouchEvent(false);
|
||||
} else if (action == MotionEvent.ACTION_CANCEL) {
|
||||
cancels++;
|
||||
getParent().requestDisallowInterceptTouchEvent(false);
|
||||
} else if (action == MotionEvent.ACTION_MOVE) {
|
||||
moves++;
|
||||
} else if (action == MotionEvent.ACTION_POINTER_DOWN) {
|
||||
pointerDowns++;
|
||||
} else if (action == MotionEvent.ACTION_POINTER_UP) {
|
||||
pointerUps++;
|
||||
}
|
||||
int count = event.getPointerCount();
|
||||
maxPointers = Math.max(maxPointers, count);
|
||||
if (count >= 2) {
|
||||
double dx = event.getX(1) - event.getX(0);
|
||||
double dy = event.getY(1) - event.getY(0);
|
||||
double distance = Math.hypot(dx, dy);
|
||||
double angle = Math.toDegrees(Math.atan2(dy, dx));
|
||||
if (initialDistance == 0 || action == MotionEvent.ACTION_POINTER_DOWN) {
|
||||
initialDistance = Math.max(distance, 0.001);
|
||||
initialAngle = angle;
|
||||
}
|
||||
scale = distance / initialDistance;
|
||||
rotation = angle - initialAngle;
|
||||
while (rotation > 180) rotation -= 360;
|
||||
while (rotation < -180) rotation += 360;
|
||||
}
|
||||
try {
|
||||
JSONArray pointers = new JSONArray();
|
||||
for (int i = 0; i < count; i++) {
|
||||
pointers.put(new JSONObject().put("id", event.getPointerId(i))
|
||||
.put("x", event.getX(i)).put("y", event.getY(i)));
|
||||
}
|
||||
if (events.length() >= 64) events.remove(0);
|
||||
events.put(new JSONObject().put("action", action).put("time_ms", event.getEventTime())
|
||||
.put("pointer_count", count).put("pointers", pointers));
|
||||
} catch (Exception ignored) {}
|
||||
publish();
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override protected void onDraw(Canvas canvas) {
|
||||
super.onDraw(canvas);
|
||||
canvas.drawText("Touch fixture: pinch, rotate, continuous stroke", dp(8), dp(30), paint);
|
||||
canvas.drawText("Down " + downs + " / up " + ups + " / moves " + moves,
|
||||
dp(8), dp(60), paint);
|
||||
canvas.drawText("Pointers " + maxPointers + " / scale " + String.format(java.util.Locale.ROOT, "%.3f", scale),
|
||||
dp(8), dp(90), paint);
|
||||
canvas.drawText("Rotation " + String.format(java.util.Locale.ROOT, "%.1f", rotation), dp(8), dp(120), paint);
|
||||
}
|
||||
|
||||
@Override public void onInitializeAccessibilityNodeInfo(AccessibilityNodeInfo info) {
|
||||
super.onInitializeAccessibilityNodeInfo(info);
|
||||
info.setClassName("com.artemis.fixture.TouchSurface");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,298 @@
|
||||
package com.artemis.helper;
|
||||
|
||||
import android.accessibilityservice.AccessibilityService;
|
||||
import android.os.Build;
|
||||
import android.os.Handler;
|
||||
import android.os.Looper;
|
||||
import android.os.SystemClock;
|
||||
import org.json.JSONObject;
|
||||
|
||||
import java.lang.reflect.Field;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.Future;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
|
||||
import static com.artemis.helper.TestMain.*;
|
||||
|
||||
/** Offline scheduler/ownership tests against the real two production classes. */
|
||||
public final class OwnershipCases {
|
||||
private OwnershipCases() {}
|
||||
|
||||
private static void scenario(String name) {
|
||||
TestMain.scenario = name;
|
||||
}
|
||||
|
||||
private static void flushMain() throws Exception {
|
||||
CountDownLatch flushed = new CountDownLatch(1);
|
||||
new Handler(Looper.getMainLooper()).post(flushed::countDown);
|
||||
check(flushed.await(2, TimeUnit.SECONDS), "main queue drained");
|
||||
}
|
||||
|
||||
private static final class MainBlock implements AutoCloseable {
|
||||
final CountDownLatch release = new CountDownLatch(1);
|
||||
|
||||
MainBlock() throws Exception {
|
||||
CountDownLatch entered = new CountDownLatch(1);
|
||||
new Handler(Looper.getMainLooper()).post(() -> {
|
||||
entered.countDown();
|
||||
try {
|
||||
release.await();
|
||||
} catch (InterruptedException interrupted) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
});
|
||||
check(entered.await(2, TimeUnit.SECONDS), "main queue is blocked before the request");
|
||||
}
|
||||
|
||||
@Override public void close() throws Exception {
|
||||
release.countDown();
|
||||
flushMain();
|
||||
}
|
||||
}
|
||||
|
||||
private static int tombstoneCount() throws Exception {
|
||||
Field registration = TouchGestures.class.getDeclaredField("REGISTRATION");
|
||||
registration.setAccessible(true);
|
||||
Field cancelled = TouchGestures.class.getDeclaredField("CANCELLED");
|
||||
cancelled.setAccessible(true);
|
||||
synchronized (registration.get(null)) {
|
||||
return ((Map<?, ?>) cancelled.get(null)).size();
|
||||
}
|
||||
}
|
||||
|
||||
private static void awaitOwner() throws Exception {
|
||||
long end = System.currentTimeMillis() + 2000;
|
||||
while (active() == null && System.currentTimeMillis() < end) Thread.sleep(1);
|
||||
check(active() != null, "request acquired shared ownership");
|
||||
}
|
||||
|
||||
private static boolean legacy(AccessibilityService service, String kind) {
|
||||
switch (kind) {
|
||||
case "tap": return GestureController.tap(service, 100, 200, 1000);
|
||||
case "double": return GestureController.doubleTap(service, 100, 200, 500);
|
||||
case "long": return GestureController.longPress(service, 100, 200, 500, 1000);
|
||||
case "swipe": return GestureController.swipe(service, 100, 200, 200, 300, 100, 1000);
|
||||
default: throw new AssertionError("Unknown legacy test action");
|
||||
}
|
||||
}
|
||||
|
||||
private static void cancellationBeforeRegistration() throws Exception {
|
||||
scenario("cancellation precedes registration and is repeatable");
|
||||
JSONObject first = cancel("cancel-before-registration");
|
||||
code(first, "CANCEL_RECORDED", "observed");
|
||||
eq(first.get("success"), true, "unmatched cancellation acknowledged");
|
||||
code(cancel("cancel-before-registration"), "CANCEL_RECORDED", "observed");
|
||||
AccessibilityService untouched = new AccessibilityService();
|
||||
code(perform(untouched, spec("cancel-before-registration", 20, 2, 1)),
|
||||
"CANCELLED", "not_started");
|
||||
eq(untouched.calls, 0, "cancelled ID cannot dispatch when it arrives later");
|
||||
waitQuiet();
|
||||
|
||||
scenario("completed request can be cancelled repeatedly without rewriting its result");
|
||||
AccessibilityService completed = new AccessibilityService();
|
||||
JSONObject original = perform(completed, spec("completed-before-cancel", 20, 2, 1));
|
||||
code(original, "OK", "completed");
|
||||
code(cancel("completed-before-cancel"), "CANCEL_RECORDED", "observed");
|
||||
code(cancel("completed-before-cancel"), "CANCEL_RECORDED", "observed");
|
||||
code(original, "OK", "completed");
|
||||
AccessibilityService duplicate = new AccessibilityService();
|
||||
code(perform(duplicate, spec("completed-before-cancel", 20, 2, 1)),
|
||||
"CANCELLED", "not_started");
|
||||
eq(duplicate.calls, 0, "repeated cancelled ID cannot inject new input");
|
||||
}
|
||||
|
||||
private static void concurrentRegistrationAndCancellation() throws Exception {
|
||||
for (int index = 0; index < 32; index++) {
|
||||
scenario("concurrent registration/cancellation iteration " + index);
|
||||
final String id = "registration-race-" + index;
|
||||
final AccessibilityService service = new AccessibilityService();
|
||||
final CountDownLatch race = new CountDownLatch(1);
|
||||
Future<JSONObject> operation;
|
||||
try (MainBlock block = new MainBlock()) {
|
||||
operation = workers.submit(() -> {
|
||||
race.await();
|
||||
return perform(service, spec(id, 20, 2, 1));
|
||||
});
|
||||
Future<JSONObject> cancellation = workers.submit(() -> {
|
||||
race.await();
|
||||
return cancel(id);
|
||||
});
|
||||
race.countDown();
|
||||
JSONObject acknowledgement = cancellation.get(2, TimeUnit.SECONDS);
|
||||
String code = (String) acknowledgement.get("code");
|
||||
check(code.equals("CANCEL_REQUESTED") || code.equals("CANCEL_RECORDED"),
|
||||
"cancellation wins either registration ordering");
|
||||
eq(service.calls, 0, "no input before blocked main is released");
|
||||
}
|
||||
code(operation.get(2, TimeUnit.SECONDS), "CANCELLED", "not_started");
|
||||
eq(service.calls, 0, "no input after cancellation and main queue release");
|
||||
waitQuiet();
|
||||
}
|
||||
}
|
||||
|
||||
private static void boundedCancellationHistory() throws Exception {
|
||||
scenario("cancellation tombstones are bounded without unsafe eviction");
|
||||
SystemClock.advance(60001);
|
||||
AccessibilityService prune = new AccessibilityService();
|
||||
code(perform(prune, spec("prune-old-cancellations", 20, 2, 1)), "OK", "completed");
|
||||
eq(tombstoneCount(), 0, "expired tombstones pruned by registration");
|
||||
for (int index = 0; index < 70; index++) {
|
||||
code(cancel("capacity-" + index), "CANCEL_RECORDED", "observed");
|
||||
check(tombstoneCount() <= 64, "retained cancellation IDs never exceed 64");
|
||||
}
|
||||
AccessibilityService evicted = new AccessibilityService();
|
||||
code(perform(evicted, spec("capacity-0", 20, 2, 1)), "BUSY", "not_started");
|
||||
eq(evicted.calls, 0, "eviction cannot revive a still-live cancelled request");
|
||||
AccessibilityService retained = new AccessibilityService();
|
||||
code(perform(retained, spec("capacity-69", 20, 2, 1)), "CANCELLED", "not_started");
|
||||
eq(retained.calls, 0, "retained cancelled request cannot start");
|
||||
AccessibilityService fresh = new AccessibilityService();
|
||||
code(perform(fresh, spec("capacity-unrelated", 20, 2, 1)), "BUSY", "not_started");
|
||||
eq(fresh.calls, 0, "capacity barrier blocks unrelated native dispatch");
|
||||
check(!legacy(fresh, "tap"), "capacity barrier also blocks legacy input");
|
||||
eq(fresh.calls, 0, "legacy cannot bypass the capacity barrier");
|
||||
|
||||
scenario("cancellation TTL expires without a real sixty-second wait");
|
||||
SystemClock.advance(60001);
|
||||
code(perform(fresh, spec("after-cancellation-ttl", 20, 2, 1)), "OK", "completed");
|
||||
eq(tombstoneCount(), 0, "expired IDs no longer occupy bounded storage");
|
||||
check(legacy(fresh, "tap"), "legacy ownership also recovers after the expiry barrier");
|
||||
waitQuiet();
|
||||
}
|
||||
|
||||
private static void bidirectionalOwnership() throws Exception {
|
||||
scenario("active native gesture excludes every legacy gesture route");
|
||||
final AccessibilityService nativeOwner = new AccessibilityService();
|
||||
nativeOwner.dropOn = 1;
|
||||
Future<JSONObject> nativePending = workers.submit(
|
||||
() -> perform(nativeOwner, spec("native-excludes-legacy", 20, 2, 1)));
|
||||
waitCalls(nativeOwner, 1);
|
||||
flushMain();
|
||||
AccessibilityService legacyOther = new AccessibilityService();
|
||||
for (String kind : new String[]{"tap", "double", "long", "swipe"}) {
|
||||
check(!legacy(legacyOther, kind), "native ownership blocks legacy " + kind);
|
||||
}
|
||||
eq(legacyOther.calls, 0, "no legacy dispatch bypasses native ownership");
|
||||
nativeOwner.completeLast();
|
||||
code(nativePending.get(2, TimeUnit.SECONDS), "OK", "completed");
|
||||
waitQuiet();
|
||||
|
||||
for (String kind : new String[]{"tap", "double", "long", "swipe"}) {
|
||||
scenario("legacy " + kind + " excludes native gesture until callback");
|
||||
final AccessibilityService legacyOwner = new AccessibilityService();
|
||||
legacyOwner.dropOn = 1;
|
||||
Future<Boolean> pending = workers.submit(() -> legacy(legacyOwner, kind));
|
||||
waitCalls(legacyOwner, 1);
|
||||
flushMain();
|
||||
AccessibilityService nativeOther = new AccessibilityService();
|
||||
code(perform(nativeOther, spec("legacy-owner-" + kind, 20, 2, 1)), "BUSY", "not_started");
|
||||
eq(nativeOther.calls, 0, "native cannot interrupt a legacy owner");
|
||||
if (kind.equals("double")) {
|
||||
eq(legacyOwner.lastGesture.strokes.size(), 2, "double tap is one two-stroke dispatch");
|
||||
eq(legacyOwner.lastGesture.strokes.get(1).start, 160L, "second tap has the intended gap");
|
||||
Thread.sleep(100);
|
||||
check(active() != null, "double tap retains ownership through the inter-tap gap");
|
||||
}
|
||||
legacyOwner.completeLast();
|
||||
check(pending.get(2, TimeUnit.SECONDS), "legacy callback reports completion");
|
||||
eq(legacyOwner.calls, 1, "legacy action dispatched exactly once");
|
||||
waitQuiet();
|
||||
code(perform(nativeOther, spec("legacy-released-" + kind, 20, 2, 1)), "OK", "completed");
|
||||
}
|
||||
|
||||
scenario("legacy API 24 stays available while continuous gestures remain gated");
|
||||
Build.VERSION.SDK_INT = 24;
|
||||
try {
|
||||
AccessibilityService oldApi = new AccessibilityService();
|
||||
check(legacy(oldApi, "tap"), "API 24 legacy tap still works");
|
||||
code(perform(oldApi, spec("native-api24-gate", 20, 2, 1)), "UNSUPPORTED", "not_started");
|
||||
eq(oldApi.calls, 1, "unsupported continuous gesture does not dispatch");
|
||||
} finally {
|
||||
Build.VERSION.SDK_INT = 36;
|
||||
}
|
||||
}
|
||||
|
||||
private static void queuedLegacyTimeoutAndInterruption() throws Exception {
|
||||
scenario("legacy main-queued timeout cannot inject after main becomes available");
|
||||
final AccessibilityService expired = new AccessibilityService();
|
||||
try (MainBlock block = new MainBlock()) {
|
||||
Future<Boolean> pending = workers.submit(() -> GestureController.tap(expired, 100, 200, 200));
|
||||
awaitOwner();
|
||||
check(!pending.get(2, TimeUnit.SECONDS), "queued legacy wait expires");
|
||||
eq(expired.calls, 0, "timed-out queued input never started");
|
||||
check(active() == null, "definitely unstarted timed-out owner is released");
|
||||
}
|
||||
eq(expired.calls, 0, "expired runnable cannot inject after main resumes");
|
||||
check(legacy(expired, "tap"), "fresh legacy work can start after safe queued abandonment");
|
||||
|
||||
scenario("interrupted legacy wait abandons the queued runnable");
|
||||
final AccessibilityService interrupted = new AccessibilityService();
|
||||
try (MainBlock block = new MainBlock()) {
|
||||
Future<Boolean> pending = workers.submit(() -> GestureController.tap(interrupted, 100, 200, 1000));
|
||||
awaitOwner();
|
||||
check(pending.cancel(true), "legacy worker interruption delivered");
|
||||
waitQuiet();
|
||||
}
|
||||
eq(interrupted.calls, 0, "interrupted queued input cannot run later");
|
||||
}
|
||||
|
||||
private static void unknownAndLateCallbackOwnership() throws Exception {
|
||||
scenario("legacy timeout retains ownership until a late native callback proves quiet");
|
||||
final AccessibilityService late = new AccessibilityService();
|
||||
late.dropOn = 1;
|
||||
Future<Boolean> pending = workers.submit(() -> GestureController.tap(late, 100, 200, 80));
|
||||
waitCalls(late, 1);
|
||||
check(!pending.get(2, TimeUnit.SECONDS), "legacy wait times out without a callback");
|
||||
check(active() != null, "in-flight timeout keeps ownership");
|
||||
AccessibilityService other = new AccessibilityService();
|
||||
code(perform(other, spec("late-legacy-still-busy", 20, 2, 1)), "BUSY", "not_started");
|
||||
check(!legacy(other, "swipe"), "legacy route cannot bypass the same unknown owner");
|
||||
eq(other.calls, 0, "unknown input produces no replacement input");
|
||||
late.completeLast();
|
||||
waitQuiet();
|
||||
eq(late.calls, 1, "late callback never replays movement");
|
||||
code(perform(other, spec("late-legacy-released", 20, 2, 1)), "OK", "completed");
|
||||
|
||||
scenario("legacy dispatch exception retains BUSY without a callback");
|
||||
AccessibilityService unknown = new AccessibilityService();
|
||||
unknown.throwOn = 1;
|
||||
check(!legacy(unknown, "tap"), "thrown dispatch does not report completion");
|
||||
check(active() != null, "thrown dispatch remains owned");
|
||||
AccessibilityService untouched = new AccessibilityService();
|
||||
code(perform(untouched, spec("legacy-throw-busy", 20, 2, 1)), "BUSY", "not_started");
|
||||
check(!legacy(untouched, "double"), "legacy input cannot bypass dispatch uncertainty");
|
||||
eq(untouched.calls, 0, "unknown throw cannot cause replacement input");
|
||||
resetUnknown(); // Test isolation only: production has no blind ownership reset.
|
||||
|
||||
scenario("late callback after a legacy dispatch exception releases ownership");
|
||||
AccessibilityService throwsThenCompletes = new AccessibilityService();
|
||||
throwsThenCompletes.throwAndCompleteOn = 1;
|
||||
throwsThenCompletes.dropOn = 1;
|
||||
check(!legacy(throwsThenCompletes, "tap"), "exception stays false even when callback arrives later");
|
||||
check(active() != null, "throw awaits callback before releasing");
|
||||
throwsThenCompletes.completeLast();
|
||||
waitQuiet();
|
||||
eq(throwsThenCompletes.calls, 1, "callback after throw does not resend input");
|
||||
check(legacy(new AccessibilityService(), "tap"), "new input is admitted after confirmed callback");
|
||||
|
||||
scenario("uncertain native gesture excludes legacy input too");
|
||||
AccessibilityService nativeUnknown = new AccessibilityService();
|
||||
nativeUnknown.throwOn = 1;
|
||||
code(perform(nativeUnknown, spec("unknown-native-legacy-gate", 20, 2, 1)), "DISPATCH_ERROR", "unknown");
|
||||
check(active() != null, "native uncertainty retains shared ownership");
|
||||
AccessibilityService legacyUntouched = new AccessibilityService();
|
||||
check(!legacy(legacyUntouched, "tap"), "legacy cannot bypass uncertain native ownership");
|
||||
eq(legacyUntouched.calls, 0, "native uncertainty admits no legacy dispatch");
|
||||
resetUnknown();
|
||||
}
|
||||
|
||||
public static void run() throws Exception {
|
||||
cancellationBeforeRegistration();
|
||||
concurrentRegistrationAndCancellation();
|
||||
boundedCancellationHistory();
|
||||
bidirectionalOwnership();
|
||||
queuedLegacyTimeoutAndInterruption();
|
||||
unknownAndLateCallbackOwnership();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
# 原生能力隔离验收
|
||||
|
||||
夹具无网络与额外权限，包名 `com.artemis.fixture`，只记录自身控件值与触摸事件。需要已安装的 Android SDK/JDK，不会下载依赖。生成物写入 `artifacts/`，不提交设备输出。
|
||||
|
||||
1. `uv run python tests/native_actions_fixture/build.py --output artifacts/native-actions-fixture/fixture.apk`
|
||||
2. 在指定且空闲设备安装该 APK；启动 `com.artemis.fixture/.MainActivity`。
|
||||
3. 使用 ARTEMIS 探索该页面：`fixture.range`、`fixture.editor`、`fixture.scroll`、`fixture.bottom` 与 `fixture.touch`。
|
||||
4. `uv run python tests/native_actions_fixture/verify_device.py --serial SERIAL --output artifacts/native-actions-fixture/verification --repeats 5`
|
||||
5. 使用正式 Flash/Pro 复跑，并检查工具记录、APP 数值与最终验收；探索结果不能代替正式模式。
|
||||
|
||||
设备脚本获取现有设备锁，通过生产 driver/actuator 执行；会正常升级打包的辅助服务，并重建**自身测试 Activity**使计数可重复，不清空数据、不停止用户 APP。它验证精确设值、选区/替换/IME、条件等待、五轮缩放/旋转/连续接触及两轮往返滚动；软键盘关闭后再测试触摸区域。结果 JSON 保留每步回执，失败样本不删除。脚本保留夹具供取证，测试结束后由操作者恢复原 APP；需要卸载时仅卸载夹具。
|
||||
|
||||
`uv run python tests/native_actions_fixture/verify_cancellation.py --serial SERIAL --output artifacts/native-actions-fixture/cancellation` 验证先取消后注册、执行中的双指取消、实际释放及返回后没有新增触点。它要求夹具已在前台、触摸区与重置按钮完整可见、键盘关闭且无窗口遮挡；前置条件不足会写 blocked 报告，不自动修复界面。判断键盘以系统 `mInputShown` 为准，IME 内部 `mIsInputViewShown` 不能单独证明窗口可见。
|
||||
|
||||
`uv run python tests/native_actions_fixture/touch_jvm_harness.py` 在临时目录编译实际 Java 源码并模拟回调，验证取消、迟到回调与输入所有权。它不连接设备，不能证明 Android 的实际触摸交付。
|
||||
|
||||
`uv run python tests/native_actions_fixture/control_windows_jvm_harness.py` 编译生产窗口解析及原生控件实现，模拟多窗口唯一性、读失败、搜索预算、取消与写入遮挡。覆盖实际选区工具栏/手柄形状、区域空洞及多个遮挡合并；同样不操作设备。
|
||||
|
||||
构建脚本使用临时测试签名。重建后签名可能变化；如安装提示签名冲突，可卸载**这个夹具**再装，不卸载用户 APP 或 ARTEMIS 助手来绕过问题。
|
||||
@@ -0,0 +1,151 @@
|
||||
"""Build the disposable native fixture using an existing Android SDK/JDK only."""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import zipfile
|
||||
|
||||
|
||||
def build(output: Path) -> Path:
|
||||
sdk = Path(
|
||||
os.environ.get("ANDROID_HOME")
|
||||
or os.environ.get("ANDROID_SDK_ROOT")
|
||||
or Path.home() / "AppData/Local/Android/Sdk"
|
||||
)
|
||||
versions = sorted(
|
||||
(sdk / "build-tools").glob("*"),
|
||||
key=lambda p: tuple(int(v) for v in p.name.split(".") if v.isdigit()),
|
||||
)
|
||||
platforms = sorted(
|
||||
(sdk / "platforms").glob("android-*"), key=lambda p: int(p.name.split("-")[-1])
|
||||
)
|
||||
if not versions or not platforms:
|
||||
raise RuntimeError("An existing Android SDK with platform and build-tools is required.")
|
||||
tools = versions[-1]
|
||||
android = platforms[-1] / "android.jar"
|
||||
executable = ".exe" if os.name == "nt" else ""
|
||||
|
||||
def run(*args: object) -> None:
|
||||
subprocess.run([str(arg) for arg in args], check=True, timeout=90)
|
||||
|
||||
java = shutil.which("java")
|
||||
javac = shutil.which("javac")
|
||||
keytool = shutil.which("keytool")
|
||||
if not java or not javac or not keytool:
|
||||
raise RuntimeError("An existing JDK (java, javac and keytool) is required.")
|
||||
output = output.resolve()
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.TemporaryDirectory(prefix="artemis-native-fixture-") as temporary:
|
||||
scratch = Path(temporary)
|
||||
classes = scratch / "classes"
|
||||
classes.mkdir()
|
||||
manifest = scratch / "AndroidManifest.xml"
|
||||
manifest.write_text(
|
||||
"""<manifest xmlns:android="http://schemas.android.com/apk/res/android" package="com.artemis.fixture">
|
||||
<application android:label="ARTEMIS Native Fixture" android:debuggable="true" android:allowBackup="false" android:theme="@android:style/Theme.Material.Light.NoActionBar">
|
||||
<activity android:name=".MainActivity" android:exported="true" android:windowSoftInputMode="adjustResize">
|
||||
<intent-filter><action android:name="android.intent.action.MAIN"/><category android:name="android.intent.category.LAUNCHER"/></intent-filter>
|
||||
</activity>
|
||||
</application>
|
||||
</manifest>""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
unsigned = scratch / "unsigned.apk"
|
||||
run(
|
||||
tools / ("aapt2" + executable),
|
||||
"link",
|
||||
"-I",
|
||||
android,
|
||||
"--manifest",
|
||||
manifest,
|
||||
"--min-sdk-version",
|
||||
26,
|
||||
"--target-sdk-version",
|
||||
35,
|
||||
"--version-code",
|
||||
1,
|
||||
"--version-name",
|
||||
"1.0",
|
||||
"-o",
|
||||
unsigned,
|
||||
)
|
||||
run(
|
||||
javac,
|
||||
"-encoding",
|
||||
"UTF-8",
|
||||
"-source",
|
||||
8,
|
||||
"-target",
|
||||
8,
|
||||
"-Xlint:-options",
|
||||
"-cp",
|
||||
android,
|
||||
"-d",
|
||||
classes,
|
||||
Path(__file__).with_name("MainActivity.java"),
|
||||
)
|
||||
run(
|
||||
java,
|
||||
"-cp",
|
||||
tools / "lib/d8.jar",
|
||||
"com.android.tools.r8.D8",
|
||||
"--lib",
|
||||
android,
|
||||
"--min-api",
|
||||
26,
|
||||
"--output",
|
||||
scratch,
|
||||
*classes.rglob("*.class"),
|
||||
)
|
||||
with zipfile.ZipFile(unsigned, "a", compression=zipfile.ZIP_DEFLATED) as archive:
|
||||
archive.write(scratch / "classes.dex", "classes.dex")
|
||||
aligned = scratch / "aligned.apk"
|
||||
run(tools / ("zipalign" + executable), "-f", "-p", 4, unsigned, aligned)
|
||||
key = scratch / "fixture.keystore"
|
||||
# Public throwaway debug signing material, never reused for distribution.
|
||||
run(
|
||||
keytool,
|
||||
"-genkeypair",
|
||||
"-keystore",
|
||||
key,
|
||||
"-storepass",
|
||||
"android",
|
||||
"-keypass",
|
||||
"android",
|
||||
"-alias",
|
||||
"fixture",
|
||||
"-keyalg",
|
||||
"RSA",
|
||||
"-validity",
|
||||
365,
|
||||
"-dname",
|
||||
"CN=ARTEMIS Test Fixture",
|
||||
"-noprompt",
|
||||
)
|
||||
run(
|
||||
java,
|
||||
"-jar",
|
||||
tools / "lib/apksigner.jar",
|
||||
"sign",
|
||||
"--ks",
|
||||
key,
|
||||
"--ks-pass",
|
||||
"pass:android",
|
||||
"--key-pass",
|
||||
"pass:android",
|
||||
"--ks-key-alias",
|
||||
"fixture",
|
||||
"--out",
|
||||
output,
|
||||
aligned,
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
print(build(parser.parse_args().output))
|
||||
@@ -0,0 +1,532 @@
|
||||
"""Explicit offline native-window integration checks; requires an existing JDK only.
|
||||
|
||||
Run with ``uv run python tests/native_actions_fixture/control_windows_jvm_harness.py``.
|
||||
Compiles the repository's actual NativeControls/ControlWindowRoots against small
|
||||
Android fakes. No device, service, network, APK, or configured model is accessed.
|
||||
Generated sources/classes live only in a self-cleaning temporary directory.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
from time import perf_counter
|
||||
|
||||
from touch_jvm_harness import find_tool, run_command
|
||||
|
||||
|
||||
STUBS = {
|
||||
"android/os/Build.java": """
|
||||
package android.os;
|
||||
public final class Build {public static final class VERSION {public static int SDK_INT=36;}}
|
||||
""",
|
||||
"android/os/Looper.java": """
|
||||
package android.os;
|
||||
public final class Looper {private static final Looper MAIN=new Looper();
|
||||
public static Looper getMainLooper(){return MAIN;}public static Looper myLooper(){return MAIN;}}
|
||||
""",
|
||||
"android/os/Handler.java": """
|
||||
package android.os;
|
||||
public final class Handler {public Handler(Looper l){}public boolean post(Runnable r){r.run();return true;}
|
||||
public void removeCallbacks(Runnable r){}}
|
||||
""",
|
||||
"android/os/Bundle.java": """
|
||||
package android.os;import java.util.*;
|
||||
public final class Bundle {private final Map<String,Object> values=new HashMap<>();
|
||||
public void putFloat(String k,float v){values.put(k,v);}public void putInt(String k,int v){values.put(k,v);}
|
||||
public void putCharSequence(String k,CharSequence v){values.put(k,v);}public Object get(String k){return values.get(k);}}
|
||||
""",
|
||||
"android/graphics/Rect.java": """
|
||||
package android.graphics;
|
||||
public final class Rect {public int left,top,right,bottom;public Rect(){}
|
||||
public Rect(int l,int t,int r,int b){left=l;top=t;right=r;bottom=b;}
|
||||
public boolean isEmpty(){return left>=right||top>=bottom;}
|
||||
public static boolean intersects(Rect a,Rect b){return a.left<b.right&&b.left<a.right&&a.top<b.bottom&&b.top<a.bottom;}
|
||||
public String flattenToString(){return left+" "+top+" "+right+" "+bottom;}
|
||||
@Override public boolean equals(Object o){if(!(o instanceof Rect))return false;Rect r=(Rect)o;
|
||||
return left==r.left&&top==r.top&&right==r.right&&bottom==r.bottom;}
|
||||
@Override public int hashCode(){return left+31*top+961*right+29791*bottom;}}
|
||||
""",
|
||||
"android/graphics/Region.java": """
|
||||
package android.graphics;import java.awt.geom.Area;import java.awt.geom.Rectangle2D;
|
||||
// Independent standard-JDK geometry implements the Android Region operations used by the real helper.
|
||||
public final class Region {
|
||||
public enum Op {INTERSECT,DIFFERENCE,UNION}
|
||||
private Area area=new Area();public Region(){}public Region(Rect r){set(r);}
|
||||
private static Area rect(Rect r){return r.isEmpty()?new Area():new Area(new Rectangle2D.Double(
|
||||
r.left,r.top,(double)r.right-r.left,(double)r.bottom-r.top));}
|
||||
public boolean set(Rect r){area=rect(r);return !isEmpty();}
|
||||
public boolean set(Region r){area=new Area(r.area);return !isEmpty();}
|
||||
public void setEmpty(){area=new Area();}public boolean isEmpty(){return area.isEmpty();}
|
||||
public boolean op(Rect r,Op op){Region other=new Region(r);return op(other,op);}
|
||||
public boolean op(Region r,Op op){Area other=new Area(r.area);
|
||||
if(op==Op.INTERSECT)area.intersect(other);else if(op==Op.DIFFERENCE)area.subtract(other);else area.add(other);
|
||||
return !isEmpty();}
|
||||
@Override public boolean equals(Object value){if(!(value instanceof Region))return false;
|
||||
Area difference=new Area(area);difference.exclusiveOr(((Region)value).area);return difference.isEmpty();}
|
||||
@Override public int hashCode(){return 0;}
|
||||
}
|
||||
""",
|
||||
"org/json/JSONArray.java": """
|
||||
package org.json;import java.util.*;
|
||||
public final class JSONArray {private final List<Object> values=new ArrayList<>();
|
||||
public JSONArray put(Object v){values.add(v);return this;}public int length(){return values.size();}
|
||||
public Object get(int i){return values.get(i);}public JSONObject getJSONObject(int i){return (JSONObject)get(i);}
|
||||
@Override public String toString(){return values.toString();}}
|
||||
""",
|
||||
"org/json/JSONObject.java": """
|
||||
package org.json;import java.util.*;
|
||||
public final class JSONObject {public static final Object NULL=new Object(){public String toString(){return "null";}};
|
||||
private final Map<String,Object> values=new LinkedHashMap<>();
|
||||
public JSONObject put(String k,Object v){values.put(k,v==null?NULL:v);return this;}
|
||||
public boolean has(String k){return values.containsKey(k);}public Object opt(String k){return values.get(k);}
|
||||
public Object get(String k){if(!has(k))throw new IllegalArgumentException(k);return opt(k);}
|
||||
public String getString(String k){return (String)get(k);}public int getInt(String k){return ((Number)get(k)).intValue();}
|
||||
public boolean getBoolean(String k){return (Boolean)get(k);}
|
||||
public JSONObject getJSONObject(String k){return (JSONObject)get(k);}
|
||||
public JSONArray getJSONArray(String k){return (JSONArray)get(k);}
|
||||
public JSONObject optJSONObject(String k){Object v=opt(k);return v instanceof JSONObject?(JSONObject)v:null;}
|
||||
public JSONArray optJSONArray(String k){Object v=opt(k);return v instanceof JSONArray?(JSONArray)v:null;}
|
||||
public boolean optBoolean(String k,boolean d){Object v=opt(k);return v instanceof Boolean?(Boolean)v:d;}
|
||||
public Iterator<String> keys(){return values.keySet().iterator();}
|
||||
@Override public String toString(){return values.toString();}}
|
||||
""",
|
||||
"android/view/accessibility/AccessibilityNodeInfo.java": """
|
||||
package android.view.accessibility;
|
||||
import android.graphics.Rect;import android.os.Bundle;import java.util.*;
|
||||
public final class AccessibilityNodeInfo {
|
||||
public static final int ACTION_CLICK=16,ACTION_LONG_CLICK=32,ACTION_FOCUS=1,ACTION_SCROLL_FORWARD=4096,
|
||||
ACTION_SCROLL_BACKWARD=8192,ACTION_SET_SELECTION=131072,ACTION_SET_TEXT=2097152;
|
||||
public static final String ACTION_ARGUMENT_PROGRESS_VALUE="progress",ACTION_ARGUMENT_SELECTION_START_INT="start",
|
||||
ACTION_ARGUMENT_SELECTION_END_INT="end",ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE="text";
|
||||
public static int live,acquired,recycled,doubleRecycle;
|
||||
public static final class Data {
|
||||
public final int identity,window;public String resource="",text="",pkg="app",cls="FrameLayout";
|
||||
public boolean packageThrows,childThrows,editable,scrollSupport,disableAfterText,visible=true,enabled=true,refresh=true;
|
||||
public int left=10,top=100,right=110,bottom=200,selectionStart=0,selectionEnd=0;
|
||||
public int actionsPerformed;public Data parent;public final List<Data> children=new ArrayList<>();
|
||||
public Data(int id,int win){identity=id;window=win;}
|
||||
public Data add(Data child){children.add(child);if(child!=null)child.parent=this;return this;}
|
||||
}
|
||||
private final Data data;private boolean released;
|
||||
public AccessibilityNodeInfo(Data d){data=d;live++;acquired++;}
|
||||
public static AccessibilityNodeInfo obtain(AccessibilityNodeInfo n){n.valid();return new AccessibilityNodeInfo(n.data);}
|
||||
private void valid(){if(released)throw new IllegalStateException("recycled node reused");}
|
||||
public void recycle(){if(released){doubleRecycle++;throw new IllegalStateException("double recycle");}
|
||||
released=true;live--;recycled++;}
|
||||
public int getWindowId(){valid();return data.window;}
|
||||
public CharSequence getPackageName(){valid();if(data.packageThrows)throw new IllegalStateException("package failed");return data.pkg;}
|
||||
public CharSequence getViewIdResourceName(){valid();return data.resource;}
|
||||
public CharSequence getClassName(){valid();return data.cls;}public CharSequence getText(){valid();return data.text;}
|
||||
public CharSequence getContentDescription(){valid();return "";}
|
||||
public int getChildCount(){valid();return data.children.size();}
|
||||
public AccessibilityNodeInfo getChild(int index){valid();if(data.childThrows)throw new IllegalStateException("child failed");
|
||||
Data d=data.children.get(index);return d==null?null:new AccessibilityNodeInfo(d);}
|
||||
public AccessibilityNodeInfo getParent(){valid();return data.parent==null?null:new AccessibilityNodeInfo(data.parent);}
|
||||
public void getBoundsInScreen(Rect r){valid();r.left=data.left;r.top=data.top;r.right=data.right;r.bottom=data.bottom;}
|
||||
public boolean refresh(){valid();return data.refresh;}
|
||||
public boolean performAction(int id,Bundle args){valid();data.actionsPerformed++;
|
||||
if(id==ACTION_SET_TEXT){data.text=(String)args.get(ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE);
|
||||
if(data.disableAfterText)data.enabled=false;}
|
||||
if(id==ACTION_SET_SELECTION){data.selectionStart=(Integer)args.get(ACTION_ARGUMENT_SELECTION_START_INT);
|
||||
data.selectionEnd=(Integer)args.get(ACTION_ARGUMENT_SELECTION_END_INT);}return true;}
|
||||
public boolean isVisibleToUser(){valid();return data.visible;}public boolean isEnabled(){valid();return data.enabled;}
|
||||
public boolean isEditable(){valid();return data.editable;}public boolean isPassword(){valid();return false;}
|
||||
public boolean isChecked(){valid();return false;}public boolean isCheckable(){valid();return false;}
|
||||
public boolean isSelected(){valid();return false;}public boolean isFocused(){valid();return false;}
|
||||
public boolean isScrollable(){valid();return true;}public boolean isShowingHintText(){valid();return false;}
|
||||
public int getTextSelectionStart(){valid();return data.selectionStart;}public int getTextSelectionEnd(){valid();return data.selectionEnd;}
|
||||
public RangeInfo getRangeInfo(){valid();return null;}
|
||||
public List<AccessibilityAction> getActionList(){valid();List<AccessibilityAction> actions=new ArrayList<>();
|
||||
actions.add(new AccessibilityAction(ACTION_CLICK));if(data.editable){actions.add(new AccessibilityAction(ACTION_SET_TEXT));
|
||||
actions.add(new AccessibilityAction(ACTION_SET_SELECTION));}if(data.scrollSupport)actions.add(new AccessibilityAction(ACTION_SCROLL_FORWARD));
|
||||
return actions;}
|
||||
@Override public boolean equals(Object o){return o instanceof AccessibilityNodeInfo
|
||||
&& data.identity==((AccessibilityNodeInfo)o).data.identity&&data.window==((AccessibilityNodeInfo)o).data.window;}
|
||||
@Override public int hashCode(){return data.identity*31+data.window;}
|
||||
public static final class RangeInfo {public static final int RANGE_TYPE_INT=0;
|
||||
public float getMin(){return 0;}public float getMax(){return 100;}public float getCurrent(){return 50;}public int getType(){return 0;}}
|
||||
public static final class AccessibilityAction {private final int id;public AccessibilityAction(int i){id=i;}public int getId(){return id;}
|
||||
public static final AccessibilityAction ACTION_SHOW_ON_SCREEN=new AccessibilityAction(16908342),
|
||||
ACTION_SET_PROGRESS=new AccessibilityAction(16908349),ACTION_SCROLL_UP=new AccessibilityAction(16908344),
|
||||
ACTION_SCROLL_DOWN=new AccessibilityAction(16908346),ACTION_SCROLL_LEFT=new AccessibilityAction(16908345),
|
||||
ACTION_SCROLL_RIGHT=new AccessibilityAction(16908347),ACTION_IME_ENTER=new AccessibilityAction(16908372);}
|
||||
}
|
||||
""",
|
||||
"android/view/accessibility/AccessibilityWindowInfo.java": """
|
||||
package android.view.accessibility;import android.graphics.Rect;import android.graphics.Region;
|
||||
public final class AccessibilityWindowInfo {
|
||||
public static final int TYPE_APPLICATION=1,TYPE_INPUT_METHOD=2,TYPE_SYSTEM=3;
|
||||
public static int live,recycled,doubleRecycle;
|
||||
public static final class Data {
|
||||
public final int id,layer;public int type=TYPE_APPLICATION,display;
|
||||
public boolean active,rootThrows,metadataThrows,boundsThrow,regionThrows;
|
||||
public int[][] regions;
|
||||
public int left=0,top=0,right=1080,bottom=2400;
|
||||
public AccessibilityNodeInfo.Data root;public Data(int i,int l,boolean a,AccessibilityNodeInfo.Data r){id=i;layer=l;active=a;root=r;}
|
||||
}
|
||||
private final Data data;private boolean released;
|
||||
public AccessibilityWindowInfo(Data d){data=d;live++;}
|
||||
private void valid(){if(released)throw new IllegalStateException("recycled window reused");
|
||||
if(data.metadataThrows)throw new IllegalStateException("metadata unavailable");}
|
||||
public int getId(){valid();return data.id;}public int getType(){valid();return data.type;}
|
||||
public int getLayer(){valid();return data.layer;}public int getDisplayId(){valid();return data.display;}
|
||||
public boolean isActive(){valid();return data.active;}
|
||||
public void getBoundsInScreen(Rect r){valid();if(data.boundsThrow)throw new IllegalStateException("bounds failed");
|
||||
r.left=data.left;r.top=data.top;r.right=data.right;r.bottom=data.bottom;}
|
||||
public void getRegionInScreen(Region r){valid();if(data.regionThrows)throw new IllegalStateException("region failed");
|
||||
if(data.regions==null){r.set(new Rect(data.left,data.top,data.right,data.bottom));return;}
|
||||
r.setEmpty();for(int[] part:data.regions)r.op(new Rect(part[0],part[1],part[2],part[3]),Region.Op.UNION);}
|
||||
public AccessibilityNodeInfo getRoot(){valid();if(data.rootThrows)throw new IllegalStateException("root failed");
|
||||
return data.root==null?null:new AccessibilityNodeInfo(data.root);}
|
||||
public void recycle(){if(released){doubleRecycle++;throw new IllegalStateException("double window recycle");}
|
||||
released=true;live--;recycled++;}
|
||||
}
|
||||
""",
|
||||
"android/accessibilityservice/AccessibilityService.java": """
|
||||
package android.accessibilityservice;
|
||||
import android.view.accessibility.*;import java.util.*;
|
||||
public final class AccessibilityService {
|
||||
public AccessibilityNodeInfo.Data active;public final List<AccessibilityWindowInfo.Data> windows=new ArrayList<>();
|
||||
public boolean windowsThrow,windowsNull,activeThrows;public int windowReads;public Runnable beforeWindowRead;
|
||||
public AccessibilityNodeInfo getRootInActiveWindow(){if(activeThrows)throw new IllegalStateException("active root failed");
|
||||
return active==null?null:new AccessibilityNodeInfo(active);}
|
||||
public List<AccessibilityWindowInfo> getWindows(){windowReads++;if(beforeWindowRead!=null)beforeWindowRead.run();
|
||||
if(windowsThrow)throw new IllegalStateException("windows unavailable");
|
||||
if(windowsNull)return null;List<AccessibilityWindowInfo> result=new ArrayList<>();
|
||||
IdentityHashMap<AccessibilityWindowInfo.Data,AccessibilityWindowInfo> seen=new IdentityHashMap<>();
|
||||
for(AccessibilityWindowInfo.Data d:windows){if(d==null){result.add(null);continue;}
|
||||
AccessibilityWindowInfo w=seen.get(d);if(w==null){w=new AccessibilityWindowInfo(d);seen.put(d,w);}result.add(w);}return result;}
|
||||
}
|
||||
""",
|
||||
"com/artemis/helper/ControlWindowsTest.java": """
|
||||
package com.artemis.helper;
|
||||
import android.accessibilityservice.AccessibilityService;import android.view.accessibility.*;
|
||||
import android.os.Build;import java.util.concurrent.atomic.AtomicBoolean;import org.json.*;
|
||||
public final class ControlWindowsTest {
|
||||
static int checks,scenarios,nextId=1;static String scenario;
|
||||
static void check(boolean v,String m){checks++;if(!v)throw new AssertionError(scenario+": "+m);}
|
||||
static void eq(Object a,Object e,String m){check(e.equals(a),m+" actual="+a+" expected="+e);}
|
||||
static AccessibilityNodeInfo.Data node(int window,String resource){AccessibilityNodeInfo.Data d=new AccessibilityNodeInfo.Data(nextId++,window);d.resource=resource;return d;}
|
||||
static AccessibilityWindowInfo.Data window(AccessibilityNodeInfo.Data root,int layer,boolean active){return new AccessibilityWindowInfo.Data(root.window,layer,active,root);}
|
||||
static AccessibilityService app(){AccessibilityService s=new AccessibilityService();s.active=node(217,"empty-shell");
|
||||
s.windows.add(window(s.active,0,true));return s;}
|
||||
static AccessibilityNodeInfo.Data add(AccessibilityService s,int id,int layer,String resource){
|
||||
AccessibilityNodeInfo.Data d=node(id,resource);s.windows.add(window(d,layer,false));return d;}
|
||||
static void rect(AccessibilityNodeInfo.Data node,int l,int t,int r,int b){node.left=l;node.top=t;node.right=r;node.bottom=b;}
|
||||
static void rect(AccessibilityWindowInfo.Data window,int l,int t,int r,int b){window.left=l;window.top=t;window.right=r;window.bottom=b;}
|
||||
static JSONObject args(String resource){return new JSONObject().put("selector",new JSONObject().put("resource_id",resource));}
|
||||
static JSONObject inspect(AccessibilityService s,String resource){return NativeControls.inspect(s,args(resource));}
|
||||
static JSONObject click(AccessibilityService s,String resource){return NativeControls.act(s,args(resource).put("action","click"));}
|
||||
static void code(JSONObject r,String expected){eq(r.getString("code"),expected,"receipt code");}
|
||||
static void begin(String name){quiet();scenario=name;scenarios++;}
|
||||
static void quiet(){check(AccessibilityNodeInfo.live==0,"all node handles recycled: "+AccessibilityNodeInfo.live);
|
||||
check(AccessibilityWindowInfo.live==0,"all window handles recycled: "+AccessibilityWindowInfo.live);
|
||||
check(AccessibilityNodeInfo.doubleRecycle==0,"no node double recycle");
|
||||
check(AccessibilityWindowInfo.doubleRecycle==0,"no window double recycle");}
|
||||
public static void main(String[] ignored){
|
||||
begin("real empty active shell with inactive upper player");AccessibilityService s=app();
|
||||
AccessibilityNodeInfo.Data player=add(s,216,1,"player");player.add(node(216,"tv_timeline_date")).add(node(216,"tab_play_or_pause"));
|
||||
JSONObject r=inspect(s,"tv_timeline_date");code(r,"OK");eq(r.getString("execution"),"observed","read only receipt");
|
||||
eq(r.getJSONObject("node").getString("resource_id"),"tv_timeline_date","upper player resolved");
|
||||
r=click(s,"tab_play_or_pause");code(r,"OK");eq(r.getString("execution"),"completed","write receipt");
|
||||
eq(player.children.get(1).actionsPerformed,1,"exactly one upper-window action");
|
||||
code(inspect(s,"empty-shell"),"OK");
|
||||
|
||||
begin("active modal excludes its lower application");s=app();s.windows.clear();s.windows.add(window(s.active,3,true));
|
||||
AccessibilityNodeInfo.Data lower=add(s,216,1,"behind-dialog");code(inspect(s,"behind-dialog"),"NOT_FOUND");
|
||||
code(click(s,"behind-dialog"),"NOT_FOUND");eq(lower.actionsPerformed,0,"no lower-window input");code(inspect(s,"empty-shell"),"OK");
|
||||
|
||||
begin("other packages, system and IME never become candidate controls");s=app();
|
||||
AccessibilityNodeInfo.Data other=add(s,216,1,"outside");other.pkg="other.app";
|
||||
AccessibilityNodeInfo.Data system=add(s,218,2,"outside");s.windows.get(2).type=AccessibilityWindowInfo.TYPE_SYSTEM;
|
||||
s.windows.get(2).rootThrows=true;AccessibilityNodeInfo.Data ime=add(s,219,3,"outside");
|
||||
s.windows.get(3).type=AccessibilityWindowInfo.TYPE_INPUT_METHOD;s.windows.get(3).rootThrows=true;
|
||||
code(click(s,"outside"),"NOT_FOUND");eq(other.actionsPerformed+system.actionsPerformed+ime.actionsPerformed,0,"no foreign input");
|
||||
|
||||
begin("different display cannot share layer ordering");s=app();add(s,216,1,"outside");s.windows.get(1).display=7;
|
||||
code(inspect(s,"outside"),"NOT_FOUND");
|
||||
|
||||
begin("API 24 window metadata remains usable");Build.VERSION.SDK_INT=24;s=app();add(s,216,1,"old-api");
|
||||
code(inspect(s,"old-api"),"OK");Build.VERSION.SDK_INT=36;
|
||||
|
||||
begin("matches across active and upper roots are globally ambiguous");s=app();s.active.resource="duplicate";
|
||||
AccessibilityNodeInfo.Data duplicate=add(s,216,1,"duplicate");r=click(s,"duplicate");code(r,"AMBIGUOUS");
|
||||
eq(r.getInt("candidate_count"),2,"global count");eq(r.getJSONArray("candidates").length(),2,"global candidates");
|
||||
eq(s.active.actionsPerformed+duplicate.actionsPerformed,0,"ambiguity performs no action");
|
||||
|
||||
begin("same-layer application still participates in uniqueness");s=app();s.active.resource="same-layer";
|
||||
add(s,216,0,"same-layer");code(inspect(s,"same-layer"),"AMBIGUOUS");
|
||||
|
||||
begin("duplicate window handles and root snapshots do not invent ambiguity");s=app();add(s,216,1,"unique");
|
||||
s.windows.add(s.windows.get(1));s.windows.add(new AccessibilityWindowInfo.Data(216,1,false,s.windows.get(1).root));
|
||||
s.windows.add(s.windows.get(0));code(inspect(s,"unique"),"OK");code(click(s,"unique"),"OK");
|
||||
eq(s.windows.get(1).root.actionsPerformed,1,"duplicate Region metadata permits only one native action");
|
||||
|
||||
begin("eligible unreadable roots invalidate an otherwise unique active match");s=app();
|
||||
add(s,216,1,"anything");s.windows.get(1).root=null;code(click(s,"empty-shell"),"READ_FAILED");
|
||||
eq(s.active.actionsPerformed,0,"unreadable candidate never permits fake uniqueness");code(inspect(s,"absent"),"READ_FAILED");
|
||||
|
||||
begin("candidate getRoot exception does not become absence");s=app();add(s,216,1,"anything");
|
||||
s.windows.get(1).rootThrows=true;code(inspect(s,"absent"),"READ_FAILED");
|
||||
|
||||
begin("candidate package missing or unreadable fails closed");s=app();AccessibilityNodeInfo.Data bad=add(s,216,1,"anything");
|
||||
bad.pkg=null;code(inspect(s,"absent"),"READ_FAILED");bad.pkg="";code(inspect(s,"absent"),"READ_FAILED");
|
||||
bad.pkg="app";bad.packageThrows=true;code(inspect(s,"absent"),"READ_FAILED");
|
||||
|
||||
begin("candidate root identity changed during discovery");s=app();add(s,216,1,"anything");
|
||||
s.windows.get(1).root=node(999,"anything");code(inspect(s,"absent"),"READ_FAILED");
|
||||
|
||||
begin("candidate child cannot be read");s=app();bad=add(s,216,1,"upper");bad.add(null);
|
||||
code(inspect(s,"absent"),"READ_FAILED");bad.children.clear();bad.add(node(216,"child"));bad.childThrows=true;
|
||||
code(inspect(s,"absent"),"READ_FAILED");
|
||||
|
||||
begin("scope resolves in upper window, target remains confined to its descendants");s=app();
|
||||
AccessibilityNodeInfo.Data scope=add(s,216,1,"container");scope.add(node(216,"inside"));s.active.add(node(217,"outside"));
|
||||
JSONObject scoped=args("inside").put("scope",new JSONObject().put("resource_id","container"));
|
||||
code(NativeControls.inspect(s,scoped),"OK");scoped.put("selector",new JSONObject().put("resource_id","outside"));
|
||||
code(NativeControls.inspect(s,scoped),"NOT_FOUND");
|
||||
|
||||
begin("scopes themselves must remain globally unique");s=app();s.active.resource="container";add(s,216,1,"container");
|
||||
code(NativeControls.inspect(s,args("inside").put("scope",new JSONObject().put("resource_id","container"))),"AMBIGUOUS");
|
||||
|
||||
begin("candidate array remains capped across windows");s=app();s.active.resource="duplicate";
|
||||
for(int i=0;i<12;i++)add(s,300+i,1,"duplicate");r=inspect(s,"duplicate");code(r,"AMBIGUOUS");
|
||||
eq(r.getInt("candidate_count"),13,"all matches counted");eq(r.getJSONArray("candidates").length(),10,"candidate snapshot cap");
|
||||
|
||||
begin("node budget is shared across roots, not reset for each window");s=app();
|
||||
AccessibilityNodeInfo.Data upper=add(s,216,1,"upper");for(int i=0;i<3000;i++){
|
||||
s.active.add(node(217,"active-child"));upper.add(node(216,"upper-child"));}
|
||||
code(inspect(s,"absent"),"LIMIT_EXCEEDED");
|
||||
|
||||
begin("search-limit failure cannot dispatch an early unique match");s.active.resource="unique";
|
||||
code(click(s,"unique"),"LIMIT_EXCEEDED");eq(s.active.actionsPerformed,0,"bounded search proves no fake uniqueness");
|
||||
|
||||
begin("window enumeration has a hard upper bound with complete cleanup");s=app();
|
||||
for(int i=0;i<65;i++)add(s,300+i,1,"upper");code(inspect(s,"absent"),"LIMIT_EXCEEDED");
|
||||
|
||||
begin("missing active root is unavailable, never absent");s=app();s.active=null;
|
||||
code(inspect(s,"absent"),"UNAVAILABLE");
|
||||
|
||||
begin("empty, missing and unavailable metadata cannot prove uniqueness or absence");s=app();s.windows.clear();
|
||||
code(inspect(s,"empty-shell"),"UNAVAILABLE");code(inspect(s,"absent"),"UNAVAILABLE");s.windowsNull=true;
|
||||
code(inspect(s,"empty-shell"),"UNAVAILABLE");s.windowsNull=false;s.windowsThrow=true;code(inspect(s,"empty-shell"),"UNAVAILABLE");
|
||||
|
||||
begin("missing active window cannot broaden to an unrelated observed application");s=app();s.windows.clear();add(s,216,1,"upper");
|
||||
code(inspect(s,"upper"),"UNAVAILABLE");code(inspect(s,"empty-shell"),"UNAVAILABLE");
|
||||
|
||||
begin("active metadata is not current");s=app();s.windows.get(0).active=false;add(s,216,1,"upper");
|
||||
code(inspect(s,"upper"),"UNAVAILABLE");
|
||||
|
||||
begin("active metadata unavailable remains conservative");s=app();s.windows.get(0).metadataThrows=true;add(s,216,1,"upper");
|
||||
code(inspect(s,"upper"),"UNAVAILABLE");
|
||||
|
||||
begin("declared candidate metadata missing after active metadata is established");s=app();s.windows.add(null);
|
||||
code(inspect(s,"absent"),"READ_FAILED");
|
||||
|
||||
begin("single known application window permits one visible native action");s=app();
|
||||
code(click(s,"empty-shell"),"OK");eq(s.active.actionsPerformed,1,"single-window input");
|
||||
|
||||
begin("invisible targets remain observable but cannot receive native actions");s=app();s.active.visible=false;
|
||||
code(inspect(s,"empty-shell"),"OK");code(click(s,"empty-shell"),"NOT_VISIBLE");
|
||||
eq(s.active.actionsPerformed,0,"invisible target never written");
|
||||
|
||||
begin("nonfocusable same-package higher modal blocks retained active root writes");s=app();add(s,216,1,"modal");
|
||||
code(inspect(s,"empty-shell"),"OK");r=click(s,"empty-shell");code(r,"OCCLUDED");
|
||||
eq(r.getString("execution"),"not_started","blocked before input");eq(s.active.actionsPerformed,0,"no modal penetration");
|
||||
|
||||
begin("any higher package or window type blocks full coverage");
|
||||
for(int type:new int[]{1,2,3}){s=app();AccessibilityNodeInfo.Data blocker=add(s,216,1,"blocker");
|
||||
blocker.pkg="other.app";s.windows.get(1).type=type;code(click(s,"empty-shell"),"OCCLUDED");
|
||||
eq(s.active.actionsPerformed,0,"foreign/system/IME overlap blocked");}
|
||||
|
||||
begin("higher nonoverlapping status bar leaves the app target actionable");s=app();
|
||||
add(s,216,1,"status-bar");s.windows.get(1).type=AccessibilityWindowInfo.TYPE_SYSTEM;s.windows.get(1).bottom=48;
|
||||
code(click(s,"empty-shell"),"OK");eq(s.active.actionsPerformed,1,"nonoverlapping system bar allowed");
|
||||
|
||||
begin("empty higher bounds do not claim occlusion");s=app();add(s,216,1,"empty-window");
|
||||
s.windows.get(1).right=0;code(click(s,"empty-shell"),"OK");
|
||||
|
||||
begin("higher windows on another display do not occlude this display");s=app();add(s,216,1,"other-display");
|
||||
s.windows.get(1).display=7;code(click(s,"empty-shell"),"OK");
|
||||
|
||||
begin("own window disappearance between resolution and action is unavailable");
|
||||
final AccessibilityService removed=app();removed.beforeWindowRead=()->{if(removed.windowReads==2)removed.windows.clear();};
|
||||
code(click(removed,"empty-shell"),"UNAVAILABLE");eq(removed.active.actionsPerformed,0,"missing context blocks input");
|
||||
|
||||
begin("a new overlapping window is checked immediately before action");
|
||||
final AccessibilityService newlyCovered=app();newlyCovered.beforeWindowRead=()->{
|
||||
if(newlyCovered.windowReads==2)add(newlyCovered,216,1,"new-modal");};
|
||||
code(click(newlyCovered,"empty-shell"),"OCCLUDED");eq(newlyCovered.active.actionsPerformed,0,"new modal blocks input");
|
||||
|
||||
begin("latest metadata read failure never permits input");
|
||||
final AccessibilityService latestFailure=app();latestFailure.beforeWindowRead=()->{
|
||||
if(latestFailure.windowReads==2)latestFailure.windowsThrow=true;};
|
||||
code(click(latestFailure,"empty-shell"),"UNAVAILABLE");eq(latestFailure.active.actionsPerformed,0,"failed latest query blocks input");
|
||||
|
||||
begin("unknown target window bounds are unavailable");s=app();s.windows.get(0).boundsThrow=true;
|
||||
code(click(s,"empty-shell"),"UNAVAILABLE");eq(s.active.actionsPerformed,0,"unknown bounds block input");
|
||||
|
||||
begin("unknown active display cannot prove complete resolution");s=app();s.windows.get(0).display=-1;
|
||||
code(inspect(s,"empty-shell"),"UNAVAILABLE");code(inspect(s,"absent"),"UNAVAILABLE");
|
||||
|
||||
begin("unknown potential occluder display cannot authorize input");s=app();add(s,216,1,"system-window");
|
||||
s.windows.get(1).type=AccessibilityWindowInfo.TYPE_SYSTEM;s.windows.get(1).display=-1;
|
||||
code(click(s,"empty-shell"),"UNAVAILABLE");eq(s.active.actionsPerformed,0,"unknown display blocks input");
|
||||
|
||||
begin("empty target or own-window bounds cannot authorize input");s=app();s.active.right=s.active.left;
|
||||
code(click(s,"empty-shell"),"UNAVAILABLE");s=app();s.windows.get(0).right=0;
|
||||
code(click(s,"empty-shell"),"UNAVAILABLE");
|
||||
|
||||
begin("post-edit caret action rechecks latest windows and retains unknown partial receipt");
|
||||
final AccessibilityService edit=app();edit.active.editable=true;edit.active.text="abc";
|
||||
edit.beforeWindowRead=()->{if(edit.windowReads==3)add(edit,216,1,"new-overlay");};
|
||||
r=NativeControls.act(edit,args("empty-shell").put("action","replace_selection").put("text","Z")
|
||||
.put("selection_start",0).put("selection_end",1));code(r,"OCCLUDED");
|
||||
eq(r.getString("execution"),"unknown","accepted text with blocked caret stays unknown");
|
||||
eq(edit.active.text,"Zbc","accepted edit retained");eq(edit.active.actionsPerformed,1,"caret not dispatched through overlay");
|
||||
|
||||
begin("real code14 editor selection toolbar and handles leave editable area exposed");
|
||||
s=new AccessibilityService();s.active=node(231,"activity");s.windows.add(window(s.active,0,true));
|
||||
rect(s.windows.get(0),0,0,720,1600);AccessibilityNodeInfo.Data editor=node(231,"editor");
|
||||
editor.editable=true;editor.text="A😀B中文";rect(editor,24,534,696,638);s.active.add(editor);
|
||||
add(s,232,4,"selection-toolbar");rect(s.windows.get(1),32,446,554,542);
|
||||
add(s,233,2,"selection-handle-left");rect(s.windows.get(2),1,611,76,691);
|
||||
add(s,234,3,"selection-handle-right");rect(s.windows.get(3),77,611,165,691);
|
||||
add(s,226,5,"status-bar");s.windows.get(4).type=AccessibilityWindowInfo.TYPE_SYSTEM;rect(s.windows.get(4),0,0,720,48);
|
||||
add(s,218,1,"keyboard");s.windows.get(5).type=AccessibilityWindowInfo.TYPE_INPUT_METHOD;rect(s.windows.get(5),0,1002,720,1600);
|
||||
r=NativeControls.act(s,args("editor").put("action","replace_selection").put("text"," X ")
|
||||
.put("selection_start",1).put("selection_end",3));code(r,"OK");
|
||||
eq(r.getString("execution"),"completed","partial occlusion permits native edit");eq(editor.text,"A X B中文","UTF-16 replacement retained");
|
||||
eq(editor.actionsPerformed,2,"text and caret each accepted");eq(editor.selectionStart,4,"caret restored after inserted whitespace");
|
||||
|
||||
begin("partial higher overlap permits a native action on the exposed part");s=app();add(s,216,1,"partial-overlay");
|
||||
rect(s.windows.get(1),0,0,60,300);code(click(s,"empty-shell"),"OK");eq(s.active.actionsPerformed,1,"exposed part remains actionable");
|
||||
|
||||
begin("multiple higher windows jointly covering the target deny input");s=app();add(s,216,1,"cover-left");
|
||||
rect(s.windows.get(1),0,0,60,300);add(s,218,2,"cover-right");rect(s.windows.get(2),60,0,200,300);
|
||||
code(click(s,"empty-shell"),"OCCLUDED");eq(s.active.actionsPerformed,0,"union coverage blocks input");
|
||||
|
||||
begin("one pixel gap in joint coverage remains exposed");s=app();add(s,216,1,"cover-left");
|
||||
rect(s.windows.get(1),0,0,60,300);add(s,218,2,"cover-right");rect(s.windows.get(2),61,0,200,300);
|
||||
code(click(s,"empty-shell"),"OK");eq(s.active.actionsPerformed,1,"integer region gap preserved");
|
||||
|
||||
begin("target area outside its own window cannot bypass full coverage inside");s=app();
|
||||
rect(s.windows.get(0),10,100,60,200);add(s,216,1,"cover-visible-half");rect(s.windows.get(1),10,100,60,200);
|
||||
code(click(s,"empty-shell"),"OCCLUDED");eq(s.active.actionsPerformed,0,"own-window clipping precedes occlusion");
|
||||
|
||||
begin("overlap only outside the own-window clip does not block exposed area");s=app();
|
||||
rect(s.windows.get(0),10,100,60,200);add(s,216,1,"outside-own-window");rect(s.windows.get(1),60,100,110,200);
|
||||
code(click(s,"empty-shell"),"OK");eq(s.active.actionsPerformed,1,"outside clip is irrelevant");
|
||||
|
||||
begin("a target completely outside its own window is unavailable");s=app();rect(s.windows.get(0),0,0,5,50);
|
||||
code(click(s,"empty-shell"),"UNAVAILABLE");eq(s.active.actionsPerformed,0,"invalid clip cannot authorize input");
|
||||
|
||||
begin("screen-sized container can scroll between system and IME edge windows");s=app();
|
||||
rect(s.active,0,0,720,1600);rect(s.windows.get(0),0,0,720,1600);s.active.scrollSupport=true;
|
||||
add(s,216,2,"status-bar");s.windows.get(1).type=AccessibilityWindowInfo.TYPE_SYSTEM;rect(s.windows.get(1),0,0,720,48);
|
||||
add(s,218,1,"keyboard");s.windows.get(2).type=AccessibilityWindowInfo.TYPE_INPUT_METHOD;rect(s.windows.get(2),0,1002,720,1600);
|
||||
code(NativeControls.act(s,args("empty-shell").put("action","scroll_forward")),"OK");
|
||||
eq(s.active.actionsPerformed,1,"visible middle permits native scroll");
|
||||
|
||||
begin("API30 own L-shaped region does not expose a target inside its bounding-box hole");Build.VERSION.SDK_INT=30;s=app();
|
||||
rect(s.active,50,100,100,150);rect(s.windows.get(0),0,0,200,300);
|
||||
s.windows.get(0).regions=new int[][]{{0,0,20,300},{0,0,200,50}};
|
||||
code(click(s,"empty-shell"),"OCCLUDED");eq(s.active.actionsPerformed,0,"own L-shape hole blocks input");Build.VERSION.SDK_INT=36;
|
||||
|
||||
begin("API30 own disconnected region cannot authorize a target in its gap");s=app();
|
||||
rect(s.active,50,100,90,200);rect(s.windows.get(0),0,0,140,300);
|
||||
s.windows.get(0).regions=new int[][]{{0,0,40,300},{100,0,140,300}};
|
||||
code(click(s,"empty-shell"),"OCCLUDED");eq(s.active.actionsPerformed,0,"own disconnected gap blocks input");
|
||||
|
||||
begin("API30 higher region hole leaves the target actionable despite bounding-box overlap");s=app();
|
||||
add(s,216,1,"hollow-overlay");rect(s.windows.get(1),0,0,200,300);
|
||||
s.windows.get(1).regions=new int[][]{{0,0,200,100},{0,200,200,300},{0,100,10,200},{110,100,200,200}};
|
||||
code(click(s,"empty-shell"),"OK");eq(s.active.actionsPerformed,1,"actual overlay hole stays exposed");
|
||||
|
||||
begin("API30 disconnected higher regions jointly covering all target area deny input");s=app();
|
||||
add(s,216,1,"alternating-a");rect(s.windows.get(1),10,100,85,200);
|
||||
s.windows.get(1).regions=new int[][]{{10,100,35,200},{60,100,85,200}};
|
||||
add(s,218,2,"alternating-b");rect(s.windows.get(2),35,100,110,200);
|
||||
s.windows.get(2).regions=new int[][]{{35,100,60,200},{85,100,110,200}};
|
||||
code(click(s,"empty-shell"),"OCCLUDED");eq(s.active.actionsPerformed,0,"union of disconnected regions blocks input");
|
||||
|
||||
begin("API29 uses conservative rectangular bounds without calling unsupported Region getter");
|
||||
Build.VERSION.SDK_INT=29;s=app();add(s,216,1,"hollow-overlay");rect(s.windows.get(1),0,0,200,300);
|
||||
s.windows.get(1).regions=new int[][]{{0,0,200,100},{0,200,200,300}};s.windows.get(1).regionThrows=true;
|
||||
s.windows.get(0).regionThrows=true;code(click(s,"empty-shell"),"OCCLUDED");Build.VERSION.SDK_INT=36;
|
||||
|
||||
begin("API24 rectangular subtraction still permits partially exposed targets");Build.VERSION.SDK_INT=24;
|
||||
s=app();add(s,216,1,"partial-overlay");rect(s.windows.get(1),0,0,60,300);
|
||||
s.windows.get(0).regionThrows=true;s.windows.get(1).regionThrows=true;
|
||||
code(click(s,"empty-shell"),"OK");Build.VERSION.SDK_INT=36;
|
||||
|
||||
begin("API30 failed Region metadata cannot silently fall back to rectangular bounds");s=app();
|
||||
s.windows.get(0).regionThrows=true;code(click(s,"empty-shell"),"UNAVAILABLE");eq(s.active.actionsPerformed,0,"own Region failure blocks input");
|
||||
s=app();add(s,216,1,"system-window");s.windows.get(1).type=AccessibilityWindowInfo.TYPE_SYSTEM;
|
||||
s.windows.get(1).regionThrows=true;code(click(s,"empty-shell"),"UNAVAILABLE");
|
||||
eq(s.active.actionsPerformed,0,"higher Region failure blocks input");
|
||||
|
||||
begin("post-edit disabled control prevents the caret action and preserves partial unknown");s=app();
|
||||
s.active.editable=true;s.active.text="abc";s.active.disableAfterText=true;
|
||||
r=NativeControls.act(s,args("empty-shell").put("action","replace_selection").put("text","Z")
|
||||
.put("selection_start",0).put("selection_end",1));code(r,"DISABLED");eq(r.getString("execution"),"unknown","disabled caret is partial");
|
||||
eq(s.active.text,"Zbc","accepted edit remains");eq(s.active.actionsPerformed,1,"disabled caret not dispatched");
|
||||
|
||||
begin("pre-cancelled discovery does not acquire input or roots");s=app();
|
||||
try(ControlWindowRoots roots=ControlWindowRoots.capture(s,new AtomicBoolean(true))){
|
||||
eq(roots.error,"CANCELLED","cancelled discovery");eq(roots.size(),0,"no root acquired");}
|
||||
quiet();System.out.println("PASS "+checks+" checks across "+scenarios+" native-window scenarios");
|
||||
}
|
||||
}
|
||||
""",
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
started = perf_counter()
|
||||
try:
|
||||
java, javac = find_tool("java"), find_tool("javac")
|
||||
helper = (
|
||||
Path(__file__).resolve().parents[2]
|
||||
/ "packages/artemis-accessibility-helper/app/src/main/java/com/artemis/helper"
|
||||
)
|
||||
actual = [helper / name for name in ("NativeControls.java", "ControlWindowRoots.java")]
|
||||
if not all(path.is_file() for path in actual):
|
||||
raise RuntimeError("The repository native-control Java sources are missing")
|
||||
print("Offline Android-window simulation against actual Java; no device input.", flush=True)
|
||||
with TemporaryDirectory(prefix="artemis-control-windows-jvm-") as temporary:
|
||||
workspace = Path(temporary)
|
||||
classes = workspace / "classes"
|
||||
classes.mkdir()
|
||||
generated = []
|
||||
for name, source in STUBS.items():
|
||||
destination = workspace / "src" / name
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
destination.write_text(source.lstrip("\n"), encoding="utf-8")
|
||||
generated.append(str(destination))
|
||||
run_command(
|
||||
[
|
||||
javac,
|
||||
"-encoding",
|
||||
"UTF-8",
|
||||
"-source",
|
||||
"8",
|
||||
"-target",
|
||||
"8",
|
||||
"-Xlint:-options",
|
||||
"-d",
|
||||
str(classes),
|
||||
*generated,
|
||||
*(str(path) for path in actual),
|
||||
],
|
||||
timeout=60,
|
||||
)
|
||||
run_command([java, "-cp", str(classes), "com.artemis.helper.ControlWindowsTest"], 30)
|
||||
print(f"Native-window harness completed in {perf_counter() - started:.2f}s.")
|
||||
return 0
|
||||
except Exception as error:
|
||||
print(f"FAIL: {error}")
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,278 @@
|
||||
"""Manual JVM simulation of TouchGestures callbacks; no Android device is used.
|
||||
|
||||
Run explicitly from the repository root:
|
||||
uv run python tests/native_actions_fixture/touch_jvm_harness.py
|
||||
|
||||
Requires a JDK (javac and java) through JAVA_HOME or PATH. Uses only the Python
|
||||
standard library, generates fake Android classes in a temporary directory, and
|
||||
compiles the repository's actual TouchGestures.java and GestureController.java.
|
||||
It checks callback ownership,
|
||||
cancellation, release, and error handling. It cannot prove real Android input,
|
||||
MotionEvent delivery, visual results, or physical pointer continuity.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from tempfile import TemporaryDirectory
|
||||
from time import perf_counter
|
||||
|
||||
|
||||
# These minimal fakes model callback scheduling, not Android's implementation.
|
||||
STUBS: dict[str, str] = {
|
||||
"android/accessibilityservice/AccessibilityService.java": r"""
|
||||
package android.accessibilityservice;
|
||||
import android.os.Handler;import android.content.Context;import java.util.*;
|
||||
public class AccessibilityService extends Context {
|
||||
public static final int GLOBAL_ACTION_BACK=1,GLOBAL_ACTION_HOME=2,GLOBAL_ACTION_RECENTS=3,GLOBAL_ACTION_NOTIFICATIONS=4,GLOBAL_ACTION_QUICK_SETTINGS=5,GLOBAL_ACTION_POWER_DIALOG=6,GLOBAL_ACTION_TOGGLE_SPLIT_SCREEN=7,GLOBAL_ACTION_LOCK_SCREEN=8,GLOBAL_ACTION_TAKE_SCREENSHOT=9;
|
||||
public volatile int calls; public volatile int rejectOn,throwOn,cancelOn,dropOn,throwAndCompleteOn;public volatile boolean duplicate,earlyStationary;
|
||||
public volatile List<GestureDescription.StrokeDescription> held;
|
||||
public volatile GestureDescription lastGesture;private volatile GestureResultCallback lastCallback;private volatile Handler lastHandler;
|
||||
public boolean performGlobalAction(int action){return false;}
|
||||
private void emit(int n,GestureDescription gesture,GestureResultCallback callback){if(n==cancelOn){held=null;callback.onCancelled(gesture);}else{held=gesture.strokes.get(0).continuing?gesture.strokes:null;callback.onCompleted(gesture);if(duplicate)callback.onCompleted(gesture);}}
|
||||
public void completeLast(){final GestureDescription gesture=lastGesture;final GestureResultCallback callback=lastCallback;lastHandler.post(()->emit(calls,gesture,callback));}
|
||||
public boolean dispatchGesture(GestureDescription gesture,GestureResultCallback callback,Handler handler){
|
||||
int n=++calls;
|
||||
if(n==rejectOn)return false;
|
||||
if(n==throwOn)throw new RuntimeException("mock dispatch uncertainty");
|
||||
for(int i=0;i<gesture.strokes.size();i++){GestureDescription.StrokeDescription s=gesture.strokes.get(i);if(s.previous!=null&&(held==null||held.get(i)!=s.previous))throw new IllegalStateException("lost continuing contact");}
|
||||
lastGesture=gesture;lastCallback=callback;lastHandler=handler;
|
||||
long duration=0;for(GestureDescription.StrokeDescription stroke:gesture.strokes)duration=Math.max(duration,stroke.start+stroke.duration);
|
||||
GestureDescription.StrokeDescription first=gesture.strokes.get(0);if(earlyStationary&&first.continuing&&first.path.x1==first.path.x2&&first.path.y1==first.path.y2)duration=0;
|
||||
if(n!=dropOn)handler.postDelayed(()->emit(n,gesture,callback),duration+3);
|
||||
if(n==throwAndCompleteOn)throw new RuntimeException("mock callback after throw");
|
||||
return true;
|
||||
}
|
||||
public static class GestureResultCallback {public void onCompleted(GestureDescription d){}public void onCancelled(GestureDescription d){}}
|
||||
}
|
||||
""",
|
||||
"android/accessibilityservice/GestureDescription.java": r"""
|
||||
package android.accessibilityservice;
|
||||
import android.graphics.Path; import java.util.*;
|
||||
public class GestureDescription {
|
||||
public final List<StrokeDescription> strokes; private GestureDescription(List<StrokeDescription> v){strokes=v;}
|
||||
public static class Builder {final List<StrokeDescription> s=new ArrayList<>(); public Builder addStroke(StrokeDescription v){s.add(v);return this;}public GestureDescription build(){return new GestureDescription(new ArrayList<>(s));}}
|
||||
public static class StrokeDescription { public final Path path;public final long start,duration; public final boolean continuing;public final StrokeDescription previous;
|
||||
public StrokeDescription(Path p,long start,long d){this(p,start,d,false);}
|
||||
public StrokeDescription(Path p,long start,long d,boolean c){this(p,start,d,c,null);if(start<0||d<1)throw new IllegalArgumentException();}
|
||||
private StrokeDescription(Path p,long start,long d,boolean c,StrokeDescription prev){path=p;this.start=start;duration=d;continuing=c;previous=prev;}
|
||||
public StrokeDescription continueStroke(Path p,long start,long d,boolean c){if(!continuing||start!=0||d<1||p.x1!=path.x2||p.y1!=path.y2)throw new IllegalArgumentException("invalid continuation");return new StrokeDescription(p,start,d,c,this);}
|
||||
}
|
||||
}
|
||||
""",
|
||||
"android/graphics/Path.java": r"""
|
||||
package android.graphics; public class Path { public float x1,y1,x2,y2; public void moveTo(float x,float y){x1=x2=x;y1=y2=y;}public void lineTo(float x,float y){x2=x;y2=y;} }
|
||||
""",
|
||||
"android/os/Build.java": r"""
|
||||
package android.os; public final class Build { public static final class VERSION { public static int SDK_INT=36; } public static final class VERSION_CODES {public static final int O=26,P=28;} }
|
||||
""",
|
||||
"android/os/Handler.java": r"""
|
||||
package android.os;
|
||||
import java.util.*; import java.util.concurrent.*;
|
||||
public class Handler {
|
||||
public static final ScheduledThreadPoolExecutor EXEC=new ScheduledThreadPoolExecutor(1,r->{Thread t=new Thread(r,"mock-main");t.setDaemon(true);return t;});
|
||||
private static final Map<Runnable,List<Future<?>>> TASKS=new ConcurrentHashMap<>();
|
||||
public Handler(Looper l){} public boolean post(Runnable r){return postDelayed(r,0);}
|
||||
public boolean postDelayed(Runnable r,long delay){Future<?> f=EXEC.schedule(r,Math.max(0,delay),TimeUnit.MILLISECONDS);TASKS.computeIfAbsent(r,k->new CopyOnWriteArrayList<>()).add(f);return true;}
|
||||
public void removeCallbacks(Runnable r){List<Future<?>> fs=TASKS.remove(r);if(fs!=null)for(Future<?> f:fs)f.cancel(false);}
|
||||
}
|
||||
""",
|
||||
"android/os/Looper.java": r"""
|
||||
package android.os; public final class Looper { private static final Looper MAIN=new Looper(); public static Looper getMainLooper(){return MAIN;} public static Looper myLooper(){return Thread.currentThread().getName().equals("mock-main")?MAIN:null;} }
|
||||
""",
|
||||
"android/os/SystemClock.java": r"""
|
||||
package android.os; public final class SystemClock { private static volatile long offset;public static long uptimeMillis(){return System.nanoTime()/1000000L+offset;}public static void advance(long millis){offset+=millis;} }
|
||||
""",
|
||||
"android/os/Bundle.java": r"""
|
||||
package android.os;public class Bundle {public void putCharSequence(String key,CharSequence value){}}
|
||||
""",
|
||||
"android/content/Context.java": r"""
|
||||
package android.content;public class Context {public static final String CLIPBOARD_SERVICE="clipboard";public Object getSystemService(String name){return null;}}
|
||||
""",
|
||||
"android/content/ClipData.java": r"""
|
||||
package android.content;public class ClipData {public static ClipData newPlainText(CharSequence label,CharSequence text){return new ClipData();}}
|
||||
""",
|
||||
"android/content/ClipboardManager.java": r"""
|
||||
package android.content;public class ClipboardManager {public void setPrimaryClip(ClipData clip){}}
|
||||
""",
|
||||
"android/util/Log.java": r"""
|
||||
package android.util;public final class Log {public static int w(String tag,String message){return 0;}public static int w(String tag,String message,Throwable error){return 0;}}
|
||||
""",
|
||||
"android/view/accessibility/AccessibilityNodeInfo.java": r"""
|
||||
package android.view.accessibility;import android.os.Bundle;public class AccessibilityNodeInfo {public static final int ACTION_SET_TEXT=2097152;public static final String ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE="text";public CharSequence getText(){return null;}public boolean isShowingHintText(){return false;}public boolean performAction(int action,Bundle args){return false;}}
|
||||
""",
|
||||
"com/artemis/helper/HierarchyDumper.java": r"""
|
||||
package com.artemis.helper;import android.accessibilityservice.AccessibilityService;import android.view.accessibility.AccessibilityNodeInfo;public final class HierarchyDumper {public static AccessibilityNodeInfo findInputNode(AccessibilityService service){return null;}public static void safeRecycle(AccessibilityNodeInfo node){}}
|
||||
""",
|
||||
"com/artemis/helper/DisplayUtils.java": r"""
|
||||
package com.artemis.helper;import android.accessibilityservice.AccessibilityService;public final class DisplayUtils {public static volatile int width=1080,height=2400;public static class DisplayInfo {public int width,height;DisplayInfo(int w,int h){width=w;height=h;}}public static DisplayInfo getDisplayInfo(AccessibilityService s){return new DisplayInfo(width,height);}}
|
||||
""",
|
||||
"com/artemis/helper/NativeControls.java": r"""
|
||||
package com.artemis.helper;import android.accessibilityservice.AccessibilityService;import org.json.JSONObject;public final class NativeControls {public static volatile JSONObject result;public static JSONObject inspect(AccessibilityService s,JSONObject args){return result;}}
|
||||
""",
|
||||
"com/artemis/helper/TestMain.java": r"""
|
||||
package com.artemis.helper;
|
||||
import android.accessibilityservice.AccessibilityService;import android.os.*;import org.json.*;import java.lang.reflect.*;import java.util.concurrent.*;import java.util.concurrent.atomic.*;
|
||||
public final class TestMain {
|
||||
static int checks;static String scenario="initialization";static ExecutorService workers=Executors.newCachedThreadPool();
|
||||
static void check(boolean v,String message){checks++;if(!v)throw new AssertionError(scenario+": "+message);}
|
||||
static void eq(Object actual,Object expected,String message){check(expected.equals(actual),message+": "+actual+" != "+expected);}
|
||||
static JSONObject spec(String id,int interval,int frames,int fingers)throws Exception {JSONArray fs=new JSONArray();for(int f=0;f<frames;f++){JSONArray p=new JSONArray();for(int n=0;n<fingers;n++)p.put(new JSONArray().put(100+n*100+f*10).put(200+n*100+f*20));fs.put(new JSONObject().put("at_ms",f*interval).put("points",p));}return new JSONObject().put("operation_id",id).put("screen_size",new JSONArray().put(1080).put(2400)).put("frames",fs);}
|
||||
static void code(JSONObject r,String c,String execution)throws Exception{eq(r.get("code"),c,"code");eq(r.get("execution"),execution,"execution");}
|
||||
static JSONObject cancel(String id)throws Exception{return TouchGestures.cancel(new JSONObject().put("operation_id",id));}
|
||||
static void waitCalls(AccessibilityService service,int calls)throws Exception {long end=System.currentTimeMillis()+2000;while(service.calls<calls&&System.currentTimeMillis()<end)Thread.sleep(1);check(service.calls>=calls,"dispatch started");}
|
||||
static Object active()throws Exception{Field f=TouchGestures.class.getDeclaredField("ACTIVE");f.setAccessible(true);return ((AtomicReference<?>)f.get(null)).get();}
|
||||
static void waitQuiet()throws Exception{long end=System.currentTimeMillis()+2000;while(active()!=null&&System.currentTimeMillis()<end)Thread.sleep(1);check(active()==null,"active ownership released");}
|
||||
static void resetUnknown()throws Exception{Field f=TouchGestures.class.getDeclaredField("ACTIVE");f.setAccessible(true);((AtomicReference<?>)f.get(null)).set(null);}
|
||||
static JSONObject perform(AccessibilityService s,JSONObject a){return TouchGestures.perform(s,a);}
|
||||
public static void main(String[] args)throws Exception {
|
||||
try{
|
||||
scenario="normal synchronized continuation";AccessibilityService s=new AccessibilityService();JSONObject r=perform(s,spec("normal",30,4,2));code(r,"OK","completed");eq(r.get("completed_segments"),3,"completed");eq(r.get("dispatched_segments"),3,"dispatched");eq(r.get("fingers"),2,"fingers");check(s.held==null,"normal final release");waitQuiet();
|
||||
scenario="duplicate callbacks";s=new AccessibilityService();s.duplicate=true;r=perform(s,spec("duplicate",20,3,1));code(r,"OK","completed");eq(s.calls,2,"duplicate callback no extra dispatch");eq(r.get("completed_segments"),2,"duplicate callback no extra completion");waitQuiet();
|
||||
scenario="Android early stationary callback preserves hold";s=new AccessibilityService();s.earlyStationary=true;JSONObject hold=spec("hold-timing",250,4,1);hold.optJSONArray("frames").optJSONObject(1).put("points",new JSONArray().put(new JSONArray().put(100).put(200)));long holdStarted=SystemClock.uptimeMillis();r=perform(s,hold);code(r,"OK","completed");check(SystemClock.uptimeMillis()-holdStarted>=750,"hold is not compressed by callback");eq(s.calls,3,"no extra gesture during hold");check(s.held==null,"hold final release");waitQuiet();
|
||||
scenario="invalid frame validation";s=new AccessibilityService();JSONObject invalid=spec("invalid",20,2,1);invalid.put("frames",new JSONArray().put(new JSONObject().put("at_ms",0).put("points",new JSONArray().put(new JSONArray().put(0).put(0)))).put(new JSONObject().put("at_ms",0).put("points",new JSONArray().put(new JSONArray().put(1).put(1)))));r=perform(s,invalid);code(r,"INVALID_ARGUMENT","not_started");eq(s.calls,0,"invalid untouched");
|
||||
scenario="display-size mismatch";DisplayUtils.width=800;r=perform(s,spec("size",20,2,1));code(r,"DISPLAY_CHANGED","not_started");eq(s.calls,0,"size untouched");DisplayUtils.width=1080;
|
||||
scenario="unsupported Android API";Build.VERSION.SDK_INT=24;r=perform(s,spec("api",20,2,1));code(r,"UNSUPPORTED","not_started");Build.VERSION.SDK_INT=36;
|
||||
scenario="ambiguous visible guard";NativeControls.result=new JSONObject().put("success",false).put("code","AMBIGUOUS");JSONObject guarded=spec("guard",20,2,1).put("require_visible","fixture:id/touch");r=perform(s,guarded);code(r,"AMBIGUOUS","not_started");eq(s.calls,0,"guard untouched");waitQuiet();
|
||||
scenario="visible guard acceptance";NativeControls.result=new JSONObject().put("success",true).put("node",new JSONObject().put("visible",true).put("enabled",true));r=perform(s,spec("guarded-ok",20,2,1).put("require_visible","fixture:id/touch"));code(r,"OK","completed");waitQuiet();
|
||||
scenario="busy ownership and mid-segment cancellation";final AccessibilityService moving=new AccessibilityService();Future<JSONObject> pending=workers.submit(()->perform(moving,spec("cancel-mid",150,4,2)));waitCalls(moving,1);AccessibilityService other=new AccessibilityService();code(perform(other,spec("busy",20,2,1)),"BUSY","not_started");eq(other.calls,0,"busy untouched");code(cancel("wrong-id"),"CANCEL_RECORDED","observed");code(cancel("cancel-mid"),"CANCEL_REQUESTED","unknown");r=pending.get(2,TimeUnit.SECONDS);code(r,"CANCELLED","cancelled");eq(r.get("completed_segments"),1,"current interval completed");eq(r.get("dispatched_segments"),1,"later movement skipped");eq(moving.calls,2,"one endpoint release");check(moving.held==null,"cancel final release");waitQuiet();
|
||||
scenario="queued cancellation before input";CountDownLatch block=new CountDownLatch(1),entered=new CountDownLatch(1);new Handler(Looper.getMainLooper()).post(()->{entered.countDown();try{block.await();}catch(Exception e){throw new RuntimeException(e);}});entered.await();final AccessibilityService queued=new AccessibilityService();pending=workers.submit(()->perform(queued,spec("queued",30,2,1)));long e=System.currentTimeMillis()+1000;while(active()==null&&System.currentTimeMillis()<e)Thread.sleep(1);cancel("queued");block.countDown();r=pending.get(2,TimeUnit.SECONDS);code(r,"CANCELLED","not_started");eq(queued.calls,0,"queued cancellation no input");waitQuiet();
|
||||
scenario="first dispatch rejection";s=new AccessibilityService();s.rejectOn=1;r=perform(s,spec("rejected",20,2,1));code(r,"DISPATCH_REJECTED","not_started");eq(r.get("dispatched_segments"),0,"rejected receipt");waitQuiet();
|
||||
scenario="Android cancellation callback";s=new AccessibilityService();s.cancelOn=1;r=perform(s,spec("os-cancel",20,3,1));code(r,"GESTURE_CANCELLED","cancelled");eq(r.get("completed_segments"),0,"OS cancelled incomplete");waitQuiet();
|
||||
scenario="continuation rejection and endpoint release";s=new AccessibilityService();s.rejectOn=2;r=perform(s,spec("reject-continuation",20,3,2));code(r,"DISPATCH_REJECTED","cancelled");eq(s.calls,3,"rejection release once");eq(r.get("completed_segments"),1,"rejected no fake completion");check(s.held==null,"rejection release confirmed");waitQuiet();
|
||||
scenario="continuation exception and endpoint release";s=new AccessibilityService();s.throwOn=2;r=perform(s,spec("throw-continuation",20,3,2));code(r,"DISPATCH_ERROR","unknown");eq(s.calls,3,"throw release once");check(s.held==null,"throw release confirmed");waitQuiet();
|
||||
scenario="late callback after dispatch exception";s=new AccessibilityService();s.throwAndCompleteOn=1;r=perform(s,spec("late-callback",60,3,1));code(r,"DISPATCH_ERROR","unknown");check(active()!=null,"unknown keeps active while in flight");code(perform(new AccessibilityService(),spec("still-busy",20,2,1)),"BUSY","not_started");waitQuiet();check(s.held==null,"late callback release confirmed");eq(s.calls,2,"no movement retry after throw");
|
||||
scenario="failed release retains ownership";final AccessibilityService releaseReject=new AccessibilityService();releaseReject.rejectOn=2;pending=workers.submit(()->perform(releaseReject,spec("release-reject",80,3,1)));waitCalls(releaseReject,1);cancel("release-reject");r=pending.get(2,TimeUnit.SECONDS);code(r,"RELEASE_REJECTED","unknown");check(active()!=null,"release failure retains active");code(perform(new AccessibilityService(),spec("blocked",20,2,1)),"BUSY","not_started");eq(releaseReject.calls,2,"release no retry");resetUnknown();
|
||||
scenario="missing callback bounded timeout";s=new AccessibilityService();s.dropOn=1;long before=SystemClock.uptimeMillis();r=perform(s,spec("timeout",15,2,1));code(r,"TIMEOUT","unknown");long elapsed=SystemClock.uptimeMillis()-before;check(elapsed<5200,"bounded timeout "+elapsed);check(active()!=null,"callback timeout retains active");eq(s.calls,1,"callback timeout no retry");resetUnknown();
|
||||
int baseline=checks;if(baseline<85)throw new AssertionError("Existing baseline checks were lost: "+baseline);
|
||||
OwnershipCases.run();
|
||||
System.out.println("BASELINE "+baseline+" existing checks; "+(checks-baseline)+" additional ownership checks");
|
||||
System.out.println("PASS "+checks+" deterministic checks");
|
||||
}finally{workers.shutdownNow();Handler.EXEC.shutdownNow();}
|
||||
}
|
||||
}
|
||||
""",
|
||||
"org/json/JSONArray.java": r"""
|
||||
package org.json;import java.util.*;public class JSONArray {private final List<Object> v=new ArrayList<>();public JSONArray put(Object x){v.add(x);return this;}public int length(){return v.size();}public Object get(int i)throws Exception{return v.get(i);}public JSONObject optJSONObject(int i){Object x=v.get(i);return x instanceof JSONObject?(JSONObject)x:null;}public JSONArray optJSONArray(int i){Object x=v.get(i);return x instanceof JSONArray?(JSONArray)x:null;}public String toString(){return v.toString();}}
|
||||
""",
|
||||
"org/json/JSONObject.java": r"""
|
||||
package org.json;import java.util.*;
|
||||
public class JSONObject {
|
||||
public static final Object NULL=new Object(){public String toString(){return "null";}};private final Map<String,Object> m=new LinkedHashMap<>();
|
||||
public JSONObject put(String k,Object v)throws Exception{m.put(k,v==null?NULL:v);return this;}public boolean has(String k){return m.containsKey(k);}public Object opt(String k){return m.get(k);}public Object get(String k)throws Exception{if(!m.containsKey(k))throw new Exception(k);return m.get(k);}
|
||||
public JSONObject optJSONObject(String k){Object v=m.get(k);return v instanceof JSONObject?(JSONObject)v:null;}public JSONArray optJSONArray(String k){Object v=m.get(k);return v instanceof JSONArray?(JSONArray)v:null;}
|
||||
public boolean optBoolean(String k){return optBoolean(k,false);}public boolean optBoolean(String k,boolean d){Object v=m.get(k);return v instanceof Boolean?(Boolean)v:d;}
|
||||
public String optString(String k,String d){Object v=m.get(k);return v instanceof String?(String)v:d;}public String toString(){return m.toString();}
|
||||
}
|
||||
""",
|
||||
}
|
||||
|
||||
|
||||
def find_tool(name: str) -> str:
|
||||
java_home = os.environ.get("JAVA_HOME")
|
||||
if java_home:
|
||||
for suffix in (".exe", ""):
|
||||
candidate = Path(java_home) / "bin" / f"{name}{suffix}"
|
||||
if candidate.is_file():
|
||||
return str(candidate)
|
||||
resolved = shutil.which(name)
|
||||
if resolved:
|
||||
return resolved
|
||||
raise RuntimeError(f"{name} was not found; set JAVA_HOME to a JDK or add it to PATH")
|
||||
|
||||
|
||||
def run_command(arguments: list[str], timeout: int) -> subprocess.CompletedProcess[str]:
|
||||
result = subprocess.run(
|
||||
arguments,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
timeout=timeout,
|
||||
check=False,
|
||||
)
|
||||
if result.stdout:
|
||||
print(result.stdout, end="" if result.stdout.endswith("\n") else "\n")
|
||||
if result.stderr:
|
||||
print(result.stderr, end="" if result.stderr.endswith("\n") else "\n", file=sys.stderr)
|
||||
if result.returncode:
|
||||
raise RuntimeError(f"{Path(arguments[0]).name} exited with status {result.returncode}")
|
||||
return result
|
||||
|
||||
|
||||
def main() -> int:
|
||||
started = perf_counter()
|
||||
try:
|
||||
javac = find_tool("javac")
|
||||
java = find_tool("java")
|
||||
repository = Path(__file__).resolve().parents[2]
|
||||
actual_source = (
|
||||
repository
|
||||
/ "packages"
|
||||
/ "artemis-accessibility-helper"
|
||||
/ "app/src/main/java/com/artemis/helper/TouchGestures.java"
|
||||
)
|
||||
if not actual_source.is_file():
|
||||
raise RuntimeError("The repository TouchGestures.java source is missing")
|
||||
legacy_source = actual_source.with_name("GestureController.java")
|
||||
ownership_cases = Path(__file__).with_name("OwnershipCases.java")
|
||||
if not legacy_source.is_file() or not ownership_cases.is_file():
|
||||
raise RuntimeError(
|
||||
"The real GestureController.java or ownership regression cases are missing"
|
||||
)
|
||||
print(
|
||||
"JVM simulation with fake Android callbacks; no device input is exercised.", flush=True
|
||||
)
|
||||
with TemporaryDirectory(prefix="artemis-touch-jvm-") as temporary:
|
||||
workspace = Path(temporary)
|
||||
sources = workspace / "src"
|
||||
classes = workspace / "classes"
|
||||
classes.mkdir()
|
||||
generated: list[str] = []
|
||||
for name, source in STUBS.items():
|
||||
destination = sources / name
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
destination.write_text(source.lstrip("\n"), encoding="utf-8")
|
||||
generated.append(str(destination))
|
||||
run_command(
|
||||
[
|
||||
javac,
|
||||
"-J-Duser.language=en",
|
||||
"-encoding",
|
||||
"UTF-8",
|
||||
"-source",
|
||||
"8",
|
||||
"-target",
|
||||
"8",
|
||||
"-Xlint:-options",
|
||||
"-d",
|
||||
str(classes),
|
||||
*generated,
|
||||
str(actual_source),
|
||||
str(legacy_source),
|
||||
str(ownership_cases),
|
||||
],
|
||||
timeout=45,
|
||||
)
|
||||
result = run_command(
|
||||
[java, "-Duser.language=en", "-cp", str(classes), "com.artemis.helper.TestMain"],
|
||||
timeout=45,
|
||||
)
|
||||
if not re.search(
|
||||
r"^PASS [1-9]\d* deterministic checks$", result.stdout, flags=re.MULTILINE
|
||||
):
|
||||
raise RuntimeError("The JVM process returned without a deterministic check count")
|
||||
print(f"Elapsed {perf_counter() - started:.2f}s (compile + JVM simulation)")
|
||||
return 0
|
||||
except (OSError, RuntimeError, subprocess.TimeoutExpired) as error:
|
||||
print(f"FAIL: {error}", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,560 @@
|
||||
"""Explicit device acceptance for cancellation on the already-explored fixture.
|
||||
|
||||
Run only when com.artemis.fixture is already foreground with its touch surface
|
||||
and reset button fully visible, no keyboard, and no covering window:
|
||||
uv run python tests/native_actions_fixture/verify_cancellation.py --serial SERIAL --output DIR
|
||||
|
||||
Writes DIR/result.json even on assertion or precondition failure. Uses the
|
||||
production device lease, AccessibilityClient connection and AndroidAdbDriver
|
||||
cancellation path. Does not launch/restart the fixture, dismiss UI, build an APK,
|
||||
clear data, or replay input. --help never connects to a device.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import base64
|
||||
from copy import deepcopy
|
||||
from datetime import UTC, datetime
|
||||
from io import BytesIO
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
from threading import Lock
|
||||
import time
|
||||
from types import SimpleNamespace
|
||||
from uuid import uuid4
|
||||
|
||||
from adbutils import AdbClient
|
||||
from PIL import Image
|
||||
|
||||
from artemis.clients.accessibility_client import AccessibilityClient
|
||||
from artemis.drivers.android.adb_driver import AndroidAdbDriver
|
||||
from artemis.mcp.actuators.adb import AdbActuator
|
||||
from artemis.runtime.adb_endpoint import adb_command
|
||||
from artemis.runtime.device_lock import DeviceBusyError, DeviceExecutionLock
|
||||
|
||||
PACKAGE = "com.artemis.fixture"
|
||||
STATE_FILE = "files/native_actions_state.json"
|
||||
|
||||
|
||||
class PreconditionError(RuntimeError):
|
||||
"""Current UI cannot safely exercise this already-explored fixture."""
|
||||
|
||||
|
||||
def stamp() -> dict:
|
||||
return {"utc": datetime.now(UTC).isoformat(), "monotonic": time.monotonic()}
|
||||
|
||||
|
||||
class RecordingAccessibilityClient(AccessibilityClient):
|
||||
"""Log real receipts, including the driver's otherwise internal cancel/drain."""
|
||||
|
||||
def __init__(self, serial: str) -> None:
|
||||
super().__init__(serial)
|
||||
self.receipts: list[dict] = []
|
||||
self.receipt_lock = Lock()
|
||||
|
||||
def native_request(self, command: str, args: dict, *, timeout: float | None = None) -> dict:
|
||||
record = {
|
||||
"command": command,
|
||||
"args": deepcopy(args),
|
||||
"started": stamp(),
|
||||
"timeout": timeout,
|
||||
}
|
||||
try:
|
||||
receipt = super().native_request(command, args, timeout=timeout)
|
||||
record["receipt"] = deepcopy(receipt)
|
||||
return receipt
|
||||
except BaseException as exc:
|
||||
record["error"] = {"type": type(exc).__name__, "message": str(exc)}
|
||||
raise
|
||||
finally:
|
||||
record["ended"] = stamp()
|
||||
with self.receipt_lock:
|
||||
self.receipts.append(record)
|
||||
|
||||
def receipt_snapshot(self) -> list[dict]:
|
||||
with self.receipt_lock:
|
||||
return deepcopy(self.receipts)
|
||||
|
||||
|
||||
def adb_read(serial: str, *args: str) -> str:
|
||||
return subprocess.run(
|
||||
adb_command(["-s", serial, "shell", *args]),
|
||||
stdin=subprocess.DEVNULL,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
check=True,
|
||||
timeout=5,
|
||||
).stdout
|
||||
|
||||
|
||||
def require(condition: bool, message: str) -> None:
|
||||
if not condition:
|
||||
raise PreconditionError(message)
|
||||
|
||||
|
||||
def bounds(node: dict, width: int, height: int) -> list[int]:
|
||||
rectangle = node.get("bounds")
|
||||
require(
|
||||
isinstance(rectangle, list)
|
||||
and len(rectangle) == 4
|
||||
and all(type(value) is int for value in rectangle),
|
||||
"Native control has no exact pixel bounds; prepare the fixture manually.",
|
||||
)
|
||||
left, top, right, bottom = rectangle
|
||||
require(
|
||||
0 <= left < right <= width and 0 <= top < bottom <= height,
|
||||
"Fixture touch/reset control is partially off screen; prepare it manually.",
|
||||
)
|
||||
return rectangle
|
||||
|
||||
|
||||
def check_covering_windows(dump: str, rectangles: list[list[int]]) -> None:
|
||||
"""WindowManager dumps windows top to bottom; refuse uncertain windows above the activity."""
|
||||
windows = list(re.finditer(r"(?m)^\s*Window #\d+ (Window\{[^\n]+\}):\s*$", dump))
|
||||
require(bool(windows), "Cannot read WindowManager z-order; covering windows are unknown.")
|
||||
fixture = [
|
||||
index
|
||||
for index, window in enumerate(windows)
|
||||
if PACKAGE + "/" in window.group(1) and "MainActivity" in window.group(1)
|
||||
]
|
||||
require(len(fixture) == 1, "Fixture activity window is missing or ambiguous.")
|
||||
frame_pattern = r"\b(?:mFrame|frame)=\[(-?\d+),\s*(-?\d+)\]\[(-?\d+),\s*(-?\d+)\]"
|
||||
for index, window in enumerate(windows[: fixture[0]]):
|
||||
end = windows[index + 1].start()
|
||||
block = dump[window.end() : end]
|
||||
if "mHasSurface=false" in block:
|
||||
continue
|
||||
require(
|
||||
"mHasSurface=true" in block,
|
||||
f"Window above fixture has unknown surface state: {window.group(1)}",
|
||||
)
|
||||
visible = re.search(r"\bisOnScreen=(true|false)\b", block)
|
||||
if visible is None:
|
||||
visible = re.search(r"\bisVisible=(true|false)\b", block)
|
||||
if visible is None:
|
||||
visible = re.search(r"\bisReadyForDisplay\(\)=(true|false)\b", block)
|
||||
if (visible is not None and visible.group(1) == "false") or "mViewVisibility=0x8" in block:
|
||||
continue
|
||||
frame = re.search(frame_pattern, block)
|
||||
require(frame is not None, f"Window above fixture has unknown bounds: {window.group(1)}")
|
||||
left, top, right, bottom = map(int, frame.groups())
|
||||
overlap = any(
|
||||
left < rect[2] and right > rect[0] and top < rect[3] and bottom > rect[1]
|
||||
for rect in rectangles
|
||||
)
|
||||
require(
|
||||
not overlap, f"A visible or uncertain window covers fixture controls: {window.group(1)}"
|
||||
)
|
||||
|
||||
|
||||
def gesture_args(width: int, height: int, rectangle: list[int]) -> dict:
|
||||
left, top, right, bottom = rectangle
|
||||
require(right - left >= 160 and bottom - top >= 100, "Observed touch surface is too small.")
|
||||
cx, cy = (left + right) // 2, (top + bottom) // 2
|
||||
span = (right - left) // 6
|
||||
dx, dy = (right - left) // 12, (bottom - top) // 12
|
||||
offsets = [(0, 0), (dx, -dy), (0, dy), (-dx, -dy), (0, 0)]
|
||||
return {
|
||||
"operation_id": str(uuid4()),
|
||||
"screen_size": [width, height],
|
||||
"frames": [
|
||||
{"at_ms": index * 400, "points": [[cx - span + x, cy + y], [cx + span + x, cy + y]]}
|
||||
for index, (x, y) in enumerate(offsets)
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
async def verify(serial: str, output: Path) -> None:
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
report = {
|
||||
"serial": serial,
|
||||
"fixture_package": PACKAGE,
|
||||
"started": stamp(),
|
||||
"status": "running",
|
||||
"checks": [],
|
||||
"records": [],
|
||||
"cleanup_errors": [],
|
||||
}
|
||||
client = RecordingAccessibilityClient(serial)
|
||||
lease = DeviceExecutionLock(serial, description="Native fixture cancellation acceptance")
|
||||
connection_attempted = False
|
||||
gesture_task: asyncio.Task | None = None
|
||||
|
||||
def check(condition: bool, label: str) -> None:
|
||||
report["checks"].append({"check": label, "passed": bool(condition), "at": stamp()})
|
||||
if not condition:
|
||||
raise AssertionError(label)
|
||||
|
||||
async def state(label: str) -> dict:
|
||||
# MainActivity writes in place after MotionEvents; only JSON decoding is retried.
|
||||
for attempt in range(5):
|
||||
raw = await asyncio.to_thread(adb_read, serial, "run-as", PACKAGE, "cat", STATE_FILE)
|
||||
try:
|
||||
value = json.loads(raw)
|
||||
except json.JSONDecodeError as exc:
|
||||
report["records"].append(
|
||||
{
|
||||
"state_read": label,
|
||||
"attempt": attempt + 1,
|
||||
"decode_error": str(exc),
|
||||
"at": stamp(),
|
||||
}
|
||||
)
|
||||
if attempt == 4:
|
||||
raise
|
||||
await asyncio.sleep(0.02)
|
||||
continue
|
||||
if not isinstance(value, dict) or not isinstance(value.get("touch"), dict):
|
||||
raise ValueError("Fixture state file has no touch object.")
|
||||
report["records"].append({"state_read": label, "state": value, "at": stamp()})
|
||||
return value
|
||||
raise AssertionError("Unreachable state retry exit")
|
||||
|
||||
async def unchanged(expected: dict, label: str, duration: float) -> None:
|
||||
deadline = time.monotonic() + duration
|
||||
while True:
|
||||
observed = await state(label)
|
||||
check(observed["touch"] == expected, label)
|
||||
if time.monotonic() >= deadline:
|
||||
return
|
||||
await asyncio.sleep(min(0.05, max(0, deadline - time.monotonic())))
|
||||
|
||||
try:
|
||||
lease.acquire(blocking=False)
|
||||
connection_attempted = True
|
||||
await asyncio.to_thread(client.connect)
|
||||
session = client.session
|
||||
require(
|
||||
session is not None and session.protocol_version >= 3, "Helper protocol 3 is required."
|
||||
)
|
||||
require(
|
||||
{"inspect_control", "control_action", "touch_gesture", "cancel_gesture"}
|
||||
<= client.native_capabilities,
|
||||
"Connected helper lacks the required native capabilities; upgrade separately.",
|
||||
)
|
||||
report["helper"] = {"version": session.version_name, "code": session.version_code}
|
||||
screen = await asyncio.to_thread(client.get_screen_data)
|
||||
with Image.open(BytesIO(base64.b64decode(screen.base64))) as image:
|
||||
width, height = image.size
|
||||
report["screen_size"] = [width, height]
|
||||
driver = AndroidAdbDriver(serial, AdbClient(), client, width, height)
|
||||
actuator = AdbActuator(
|
||||
SimpleNamespace(
|
||||
device=SimpleNamespace(device_width=width, device_height=height), data_engine=None
|
||||
),
|
||||
SimpleNamespace(driver=driver, native_request=driver.native_request),
|
||||
)
|
||||
|
||||
async def tool(name: str, **args) -> dict:
|
||||
started = stamp()
|
||||
result = await getattr(actuator, name)(**args)
|
||||
report["records"].append(
|
||||
{
|
||||
"tool": name,
|
||||
"args": args,
|
||||
"result": result.model_dump(mode="json"),
|
||||
"started": started,
|
||||
"ended": stamp(),
|
||||
}
|
||||
)
|
||||
if name == "read_control":
|
||||
require(result.ok, f"Native {name} refused: {result.message}")
|
||||
else:
|
||||
check(result.ok, f"Native {name} accepted: {result.message}")
|
||||
return result.metadata
|
||||
|
||||
async def preconditions() -> tuple[dict, dict]:
|
||||
foreground = await driver.get_current_package()
|
||||
require(
|
||||
foreground == PACKAGE,
|
||||
"Fixture is not foreground; no UI will be launched or repaired.",
|
||||
)
|
||||
ime = await asyncio.to_thread(adb_read, serial, "dumpsys", "input_method")
|
||||
ime_flags = re.findall(r"\b(?:mInputShown|mIsInputViewShown)=(?:true|false)\b", ime)
|
||||
report["records"].append(
|
||||
{"foreground": foreground, "ime_flags": ime_flags, "at": stamp()}
|
||||
)
|
||||
# IME-internal mIsInputViewShown may stay true for a hidden window.
|
||||
# Only the system's unique mInputShown value establishes visibility.
|
||||
system_input_shown = re.findall(r"\bmInputShown=(true|false)\b", ime)
|
||||
require(
|
||||
system_input_shown == ["false"],
|
||||
"Keyboard is visible or visibility is unknown; prepare the fixture manually.",
|
||||
)
|
||||
nodes = []
|
||||
for desc in ("fixture.touch", "fixture.touch_reset"):
|
||||
node = (
|
||||
await tool(
|
||||
"read_control", selector={"package_name": PACKAGE, "content_desc": desc}
|
||||
)
|
||||
)["node"]
|
||||
require(
|
||||
node.get("package_name") == PACKAGE
|
||||
and node.get("visible") is True
|
||||
and node.get("enabled") is True,
|
||||
f"{desc} is not a visible enabled fixture control.",
|
||||
)
|
||||
bounds(node, width, height)
|
||||
nodes.append(node)
|
||||
require(
|
||||
"click" in nodes[1].get("actions", []), "Reset control lacks a native click action."
|
||||
)
|
||||
windows = await asyncio.to_thread(adb_read, serial, "dumpsys", "window", "windows")
|
||||
report["records"].append(
|
||||
{
|
||||
"window_diagnostics": [
|
||||
line
|
||||
for line in windows.splitlines()
|
||||
if re.search(
|
||||
r"Window #|mCurrentFocus|mHasSurface|isOnScreen|Frames:|mFrame=", line
|
||||
)
|
||||
],
|
||||
"at": stamp(),
|
||||
}
|
||||
)
|
||||
require(
|
||||
not re.search(r"\b(?:mShowingLockscreen|mDreamingLockscreen)=true\b", windows),
|
||||
"A lock screen covers the fixture.",
|
||||
)
|
||||
check_covering_windows(windows, [node["bounds"] for node in nodes])
|
||||
require(
|
||||
await driver.get_current_package() == PACKAGE,
|
||||
"Foreground changed during preflight.",
|
||||
)
|
||||
return nodes[0], nodes[1]
|
||||
|
||||
async def reset() -> tuple[dict, dict]:
|
||||
_, button = await preconditions()
|
||||
previous = (await state("before native reset"))["touch"]
|
||||
require(
|
||||
previous.get("down_count")
|
||||
== previous.get("up_count", 0) + previous.get("cancel_count", 0),
|
||||
"The fixture already has a held touch; do not reset its live counters.",
|
||||
)
|
||||
await tool(
|
||||
"control_action",
|
||||
selector={"package_name": PACKAGE, "content_desc": "fixture.touch_reset"},
|
||||
action="click",
|
||||
expected_token=button["token"],
|
||||
)
|
||||
baseline = await state("after native reset")
|
||||
check(
|
||||
all(
|
||||
baseline["touch"].get(key) == 0
|
||||
for key in (
|
||||
"down_count",
|
||||
"up_count",
|
||||
"cancel_count",
|
||||
"move_count",
|
||||
"max_pointer_count",
|
||||
"pointer_down_count",
|
||||
"pointer_up_count",
|
||||
)
|
||||
),
|
||||
"native reset cleared app touch counters",
|
||||
)
|
||||
check(baseline["touch"].get("events") == [], "native reset cleared app event history")
|
||||
# Reset publishes a status label and may relayout the activity.
|
||||
touch, _ = await preconditions()
|
||||
return touch, baseline
|
||||
|
||||
touch, baseline = await reset()
|
||||
late_args = gesture_args(width, height, touch["bounds"])
|
||||
cancelled_id = late_args["operation_id"]
|
||||
for label in ("cancel before registration", "repeated cancel before registration"):
|
||||
receipt = await driver.native_request(
|
||||
"cancel_gesture", {"operation_id": cancelled_id}, timeout=2
|
||||
)
|
||||
check(
|
||||
receipt.get("success") is True
|
||||
and receipt.get("code") == "CANCEL_RECORDED"
|
||||
and receipt.get("execution") == "observed",
|
||||
label,
|
||||
)
|
||||
receipt = await driver.native_request("touch_gesture", late_args, timeout=6.6)
|
||||
check(
|
||||
receipt.get("success") is False
|
||||
and receipt.get("code") == "CANCELLED"
|
||||
and receipt.get("execution") == "not_started"
|
||||
and receipt.get("dispatched_segments") == 0,
|
||||
"late same-ID gesture rejected before any input",
|
||||
)
|
||||
receipt = await driver.native_request(
|
||||
"cancel_gesture", {"operation_id": cancelled_id}, timeout=2
|
||||
)
|
||||
check(
|
||||
receipt.get("success") is True
|
||||
and receipt.get("code") == "CANCEL_RECORDED"
|
||||
and receipt.get("execution") == "observed",
|
||||
"cancel remains safe after rejected request",
|
||||
)
|
||||
await unchanged(
|
||||
baseline["touch"], "pre-registration cancellation leaves app untouched", 0.15
|
||||
)
|
||||
|
||||
touch, baseline = await reset()
|
||||
active_args = gesture_args(width, height, touch["bounds"])
|
||||
active_id = active_args["operation_id"]
|
||||
report["records"].append({"gesture_task_started": stamp(), "operation_id": active_id})
|
||||
gesture_task = asyncio.create_task(
|
||||
driver.native_request("touch_gesture", active_args, timeout=6.6)
|
||||
)
|
||||
down_deadline = time.monotonic() + 1.2
|
||||
while True:
|
||||
down = (await state("observe app down before Python cancellation"))["touch"]
|
||||
if down.get("down_count") == 1 and down.get("max_pointer_count") == 2:
|
||||
check(
|
||||
down.get("up_count") == 0 and down.get("cancel_count") == 0,
|
||||
"cancel only while app has an active two-finger contact",
|
||||
)
|
||||
break
|
||||
check(
|
||||
not gesture_task.done() and time.monotonic() < down_deadline,
|
||||
"app observes two-finger down before the gesture completes",
|
||||
)
|
||||
await asyncio.sleep(0.025)
|
||||
check(not gesture_task.done(), "production native request still pending at cancellation")
|
||||
cancel_at = stamp()
|
||||
report["records"].append(
|
||||
{"python_task_cancel_requested": cancel_at, "operation_id": active_id}
|
||||
)
|
||||
gesture_task.cancel()
|
||||
try:
|
||||
await gesture_task
|
||||
except asyncio.CancelledError:
|
||||
current = asyncio.current_task()
|
||||
if current is not None and current.cancelling():
|
||||
raise
|
||||
cancelled_at = stamp()
|
||||
report["records"].append(
|
||||
{"python_cancelled_error_observed": cancelled_at, "operation_id": active_id}
|
||||
)
|
||||
else:
|
||||
raise AssertionError("Production driver did not propagate CancelledError")
|
||||
receipts = [
|
||||
item
|
||||
for item in client.receipt_snapshot()
|
||||
if item["args"].get("operation_id") == active_id
|
||||
]
|
||||
original = [item for item in receipts if item["command"] == "touch_gesture"]
|
||||
cancellations = [item for item in receipts if item["command"] == "cancel_gesture"]
|
||||
check(
|
||||
len(original) == len(cancellations) == 1,
|
||||
"driver sends one gesture and one cancellation without replay",
|
||||
)
|
||||
final_receipt = original[0].get("receipt", {})
|
||||
check(
|
||||
final_receipt.get("code") == "CANCELLED"
|
||||
and final_receipt.get("execution") == "cancelled"
|
||||
and 0 < final_receipt.get("dispatched_segments", 0) < len(active_args["frames"]) - 1,
|
||||
"driver drains the real interrupted gesture receipt",
|
||||
)
|
||||
check(
|
||||
cancellations[0].get("receipt", {}).get("code") == "CANCEL_REQUESTED",
|
||||
"driver cancellation reaches the active native gesture",
|
||||
)
|
||||
check(
|
||||
original[0]["ended"]["monotonic"] <= cancelled_at["monotonic"],
|
||||
"CancelledError is observed only after original native request drains",
|
||||
)
|
||||
stopped = (await state("immediately after CancelledError"))["touch"]
|
||||
check(
|
||||
stopped.get("down_count") == stopped.get("up_count") == 1
|
||||
and stopped.get("cancel_count") == 0
|
||||
and stopped.get("max_pointer_count") == 2,
|
||||
"app received one two-finger down/up and no cancel",
|
||||
)
|
||||
check(
|
||||
stopped.get("pointer_down_count") == stopped.get("pointer_up_count") == 1,
|
||||
"both native pointers are released exactly once",
|
||||
)
|
||||
await unchanged(stopped, "no new app input for 500ms after CancelledError", 0.5)
|
||||
|
||||
touch, _ = await preconditions()
|
||||
for invalid in ("out_of_bounds", "nonincreasing_time"):
|
||||
args = gesture_args(width, height, touch["bounds"])
|
||||
if invalid == "out_of_bounds":
|
||||
args["frames"][-1]["points"][0][0] = width
|
||||
else:
|
||||
args["frames"][-1]["at_ms"] = args["frames"][-2]["at_ms"]
|
||||
receipt = await driver.native_request("touch_gesture", args, timeout=6.6)
|
||||
check(
|
||||
receipt.get("success") is False
|
||||
and receipt.get("code") == "INVALID_ARGUMENT"
|
||||
and receipt.get("execution") == "not_started"
|
||||
and receipt.get("dispatched_segments") == 0,
|
||||
f"raw {invalid} gesture rejected before input",
|
||||
)
|
||||
await unchanged(
|
||||
stopped, f"raw {invalid} rejection leaves app touch state unchanged", 0.1
|
||||
)
|
||||
report["status"] = "passed"
|
||||
except BaseException as exc:
|
||||
report["status"] = (
|
||||
"blocked"
|
||||
if isinstance(exc, (PreconditionError, DeviceBusyError))
|
||||
else "cancelled"
|
||||
if isinstance(exc, (asyncio.CancelledError, KeyboardInterrupt))
|
||||
else "failed"
|
||||
)
|
||||
report["error"] = {"type": type(exc).__name__, "message": str(exc)}
|
||||
raise
|
||||
finally:
|
||||
# Never release the device lease while the production driver still drains input.
|
||||
if gesture_task is not None and not gesture_task.done():
|
||||
gesture_task.cancel()
|
||||
while not gesture_task.done():
|
||||
try:
|
||||
await asyncio.shield(gesture_task)
|
||||
except asyncio.CancelledError:
|
||||
continue
|
||||
except Exception as exc:
|
||||
report["cleanup_errors"].append({"stage": "drain gesture", "error": str(exc)})
|
||||
break
|
||||
if connection_attempted:
|
||||
try:
|
||||
await state("finally")
|
||||
except BaseException as exc:
|
||||
report["cleanup_errors"].append({"stage": "final state", "error": str(exc)})
|
||||
try:
|
||||
await asyncio.to_thread(client.disconnect)
|
||||
except BaseException as exc:
|
||||
report["cleanup_errors"].append({"stage": "disconnect", "error": str(exc)})
|
||||
try:
|
||||
lease.release()
|
||||
except Exception as exc:
|
||||
report["cleanup_errors"].append({"stage": "lease release", "error": str(exc)})
|
||||
if report["cleanup_errors"] and report["status"] == "passed":
|
||||
report["status"] = "failed"
|
||||
report["native_receipts"] = client.receipt_snapshot()
|
||||
report["ended"] = stamp()
|
||||
report["elapsed_seconds"] = report["ended"]["monotonic"] - report["started"]["monotonic"]
|
||||
(output / "result.json").write_text(
|
||||
json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8"
|
||||
)
|
||||
if report["status"] != "passed":
|
||||
raise RuntimeError("Acceptance cleanup failed; see result.json.")
|
||||
print(
|
||||
f"PASS {len(report['checks'])} checks; receipts and app state: {output / 'result.json'}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(
|
||||
description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
|
||||
)
|
||||
parser.add_argument(
|
||||
"--serial", required=True, help="Explicit already-explored Android device serial"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output", required=True, type=Path, help="Artifact directory; report is result.json"
|
||||
)
|
||||
options = parser.parse_args()
|
||||
if not options.serial.strip():
|
||||
parser.error("--serial cannot be blank")
|
||||
asyncio.run(verify(options.serial, options.output))
|
||||
@@ -0,0 +1,445 @@
|
||||
"""Explicit device acceptance for the already-explored isolated native fixture.
|
||||
|
||||
Run after building/installing MainActivity and exploring it through ARTEMIS:
|
||||
uv run python tests/native_actions_fixture/verify_device.py --serial SERIAL --output DIR
|
||||
|
||||
Uses the existing device lease, driver and actuator. Never resets app data or
|
||||
stops another app. Artifacts contain fixture state and actual native receipts.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import base64
|
||||
from io import BytesIO
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import time
|
||||
from types import SimpleNamespace
|
||||
|
||||
from adbutils import AdbClient
|
||||
from PIL import Image
|
||||
|
||||
from artemis.clients.accessibility_client import AccessibilityClient
|
||||
from artemis.drivers.android.adb_driver import AndroidAdbDriver
|
||||
from artemis.mcp.actuators.adb import AdbActuator
|
||||
from artemis.runtime.adb_endpoint import adb_command
|
||||
from artemis.runtime.device_lock import DeviceExecutionLock
|
||||
|
||||
|
||||
async def verify(serial: str, output: Path, repeats: int) -> None:
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
lease = DeviceExecutionLock(serial, description="Native fixture acceptance")
|
||||
lease.acquire(blocking=False)
|
||||
client = AccessibilityClient(serial)
|
||||
records = []
|
||||
checks = []
|
||||
started = time.monotonic()
|
||||
|
||||
def check(condition, label):
|
||||
checks.append({"check": label, "passed": bool(condition)})
|
||||
if not condition:
|
||||
raise AssertionError(label)
|
||||
|
||||
def state():
|
||||
result = subprocess.run(
|
||||
adb_command(
|
||||
[
|
||||
"-s",
|
||||
serial,
|
||||
"shell",
|
||||
"run-as",
|
||||
"com.artemis.fixture",
|
||||
"cat",
|
||||
"files/native_actions_state.json",
|
||||
]
|
||||
),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
check=True,
|
||||
timeout=10,
|
||||
)
|
||||
return json.loads(result.stdout)
|
||||
|
||||
try:
|
||||
await asyncio.to_thread(client.connect)
|
||||
session = client.session
|
||||
check(session is not None and session.protocol_version >= 3, "helper protocol 3")
|
||||
# Recreate only this isolated test Activity, so counters are reproducible.
|
||||
for command in (
|
||||
["shell", "am", "force-stop", "com.artemis.fixture"],
|
||||
["shell", "am", "start", "-W", "-n", "com.artemis.fixture/.MainActivity"],
|
||||
):
|
||||
subprocess.run(
|
||||
adb_command(["-s", serial, *command]), capture_output=True, check=True, timeout=15
|
||||
)
|
||||
screen = await asyncio.to_thread(client.get_screen_data)
|
||||
image = Image.open(BytesIO(base64.b64decode(screen.base64)))
|
||||
width, height = image.size
|
||||
driver = AndroidAdbDriver(serial, AdbClient(), client, width, height)
|
||||
ctx = SimpleNamespace(
|
||||
device=SimpleNamespace(device_width=width, device_height=height), data_engine=None
|
||||
)
|
||||
actuator = AdbActuator(
|
||||
ctx, SimpleNamespace(driver=driver, native_request=driver.native_request)
|
||||
)
|
||||
|
||||
async def call(name, *, expect=True, **args):
|
||||
result = await getattr(actuator, name)(**args)
|
||||
records.append({"action": name, "args": args, "result": result.model_dump(mode="json")})
|
||||
check(result.ok is expect, f"{name} expected success={expect}: {result.message}")
|
||||
return result.metadata
|
||||
|
||||
async def read(desc):
|
||||
return (await call("read_control", selector={"content_desc": desc}))["node"]
|
||||
|
||||
async def control(desc, action, **args):
|
||||
# Focus/selection can asynchronously pan the window for the IME.
|
||||
# Observe a settled target instead of treating a safe stale-token
|
||||
# rejection during that animation as successful input or retrying it.
|
||||
async def settled_editor():
|
||||
await call(
|
||||
"wait_until",
|
||||
selector={"content_desc": desc},
|
||||
state="stable",
|
||||
timeout_ms=3000,
|
||||
stable_ms=500,
|
||||
poll_interval_ms=100,
|
||||
)
|
||||
|
||||
if desc == "fixture.editor":
|
||||
await settled_editor()
|
||||
receipt = await call(
|
||||
"control_action", selector={"content_desc": desc}, action=action, **args
|
||||
)
|
||||
if desc == "fixture.editor":
|
||||
# Selection handles/toolbars can disappear after a successful
|
||||
# write. Wait through incomplete window reads; never replay it.
|
||||
await settled_editor()
|
||||
return receipt
|
||||
|
||||
await read("fixture.title")
|
||||
original = state()
|
||||
records.append(
|
||||
{
|
||||
"initial_fixture_state": original,
|
||||
"helper_version": session.version_name,
|
||||
"helper_code": session.version_code,
|
||||
}
|
||||
)
|
||||
|
||||
ambiguous = await call(
|
||||
"read_control", expect=False, selector={"class_name": "android.widget.TextView"}
|
||||
)
|
||||
check(ambiguous["code"] == "AMBIGUOUS", "ambiguous selector rejected")
|
||||
missing = await call(
|
||||
"wait_until",
|
||||
selector={"content_desc": "fixture.missing"},
|
||||
state="absent",
|
||||
timeout_ms=100,
|
||||
)
|
||||
check(missing["success"], "short absent wait works")
|
||||
timeout = await call(
|
||||
"wait_until",
|
||||
expect=False,
|
||||
selector={"content_desc": "fixture.missing"},
|
||||
state="visible",
|
||||
timeout_ms=100,
|
||||
)
|
||||
check(timeout["code"] == "TIMEOUT", "missing visible times out")
|
||||
unsupported = await call(
|
||||
"control_action",
|
||||
expect=False,
|
||||
selector={"content_desc": "fixture.click"},
|
||||
action="set_progress",
|
||||
value=50,
|
||||
)
|
||||
check(unsupported["execution"] == "not_started", "unsupported node action did not start")
|
||||
|
||||
for iteration in range(repeats):
|
||||
print(
|
||||
f"fixture iteration {iteration + 1}/{repeats}: native controls/text/wait",
|
||||
flush=True,
|
||||
)
|
||||
value = [73, 21, 88, 49, 65][iteration % 5]
|
||||
node = await read("fixture.range")
|
||||
check(
|
||||
node["range"]["min"] == 0 and node["range"]["max"] == 100, "range endpoints exact"
|
||||
)
|
||||
await control(
|
||||
"fixture.range", "set_progress", value=value, expected_token=node["token"]
|
||||
)
|
||||
await call(
|
||||
"wait_until",
|
||||
selector={"content_desc": "fixture.range"},
|
||||
state="value",
|
||||
expected_value=value,
|
||||
timeout_ms=1500,
|
||||
)
|
||||
check(state()["progress"] == value, "range value persisted in app")
|
||||
before = state()
|
||||
await control("fixture.click", "click")
|
||||
await control("fixture.long_click", "long_click")
|
||||
after = state()
|
||||
check(after["click_count"] == before["click_count"] + 1, "native click delivered once")
|
||||
check(
|
||||
after["long_click_count"] == before["long_click_count"] + 1,
|
||||
"native long click delivered once",
|
||||
)
|
||||
|
||||
await control("fixture.editor", "set_text", text="A😀B中文")
|
||||
bad = await call(
|
||||
"control_action",
|
||||
expect=False,
|
||||
selector={"content_desc": "fixture.editor"},
|
||||
action="set_selection",
|
||||
selection_start=2,
|
||||
selection_end=3,
|
||||
)
|
||||
check(
|
||||
bad["execution"] == "not_started" and state()["text"] == "A😀B中文",
|
||||
"surrogate split rejected before edit",
|
||||
)
|
||||
await control("fixture.editor", "set_selection", selection_start=1, selection_end=3)
|
||||
selected = await read("fixture.editor")
|
||||
check(
|
||||
(selected["selection_start"], selected["selection_end"]) == (1, 3),
|
||||
"UTF16 emoji selection exact",
|
||||
)
|
||||
await control("fixture.editor", "replace_selection", text=" X ")
|
||||
edited = await read("fixture.editor")
|
||||
check(
|
||||
edited["text"] == "A X B中文"
|
||||
and edited["selection_start"] == edited["selection_end"] == 4,
|
||||
"replace only selection and place caret",
|
||||
)
|
||||
stale = await call(
|
||||
"control_action",
|
||||
expect=False,
|
||||
selector={"content_desc": "fixture.editor"},
|
||||
action="set_text",
|
||||
text="wrong",
|
||||
expected_token=selected["token"],
|
||||
)
|
||||
check(
|
||||
stale["code"] == "STALE" and state()["text"] == "A X B中文",
|
||||
"stale token cannot edit changed node",
|
||||
)
|
||||
await control("fixture.editor", "set_text", text=" 保留空白 😀 ")
|
||||
check(
|
||||
(await read("fixture.editor"))["text"] == " 保留空白 😀 ",
|
||||
"native set text preserves whitespace",
|
||||
)
|
||||
editor_node = await read("fixture.editor")
|
||||
if not editor_node["focused"]:
|
||||
await control("fixture.editor", "focus")
|
||||
ime_before = state()["ime_enter_count"]
|
||||
await control("fixture.editor", "ime_enter")
|
||||
check(state()["ime_enter_count"] == ime_before + 1, "IME submit delivered once")
|
||||
|
||||
ime = subprocess.run(
|
||||
adb_command(["-s", serial, "shell", "dumpsys", "input_method"]),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
check=True,
|
||||
timeout=10,
|
||||
)
|
||||
if "mInputShown=true" in ime.stdout:
|
||||
check(await asyncio.to_thread(client.press_key, "back"), "dismiss visible IME")
|
||||
deadline = time.monotonic() + 2
|
||||
while True:
|
||||
ime = subprocess.run(
|
||||
adb_command(["-s", serial, "shell", "dumpsys", "input_method"]),
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
errors="replace",
|
||||
check=True,
|
||||
timeout=10,
|
||||
)
|
||||
if "mInputShown=false" in ime.stdout or time.monotonic() >= deadline:
|
||||
break
|
||||
await asyncio.sleep(0.05)
|
||||
check("mInputShown=false" in ime.stdout, "IME no longer covers touch surface")
|
||||
await call(
|
||||
"wait_until",
|
||||
selector={"content_desc": "fixture.touch"},
|
||||
state="stable",
|
||||
stable_ms=300,
|
||||
timeout_ms=1500,
|
||||
poll_interval_ms=100,
|
||||
)
|
||||
|
||||
before = state()
|
||||
await control("fixture.delayed", "click")
|
||||
waiting = await read("fixture.status")
|
||||
expected_version = (
|
||||
int(before["status"].rsplit(":", 1)[-1]) + 1
|
||||
if before["status"].startswith("status:changed:")
|
||||
else iteration + 1
|
||||
)
|
||||
# Each delayed callback increments its own counter once per iteration.
|
||||
await call(
|
||||
"wait_until",
|
||||
selector={"content_desc": "fixture.status"},
|
||||
state="text",
|
||||
expected_text=f"status:changed:{expected_version}",
|
||||
timeout_ms=2500,
|
||||
poll_interval_ms=100,
|
||||
)
|
||||
check(
|
||||
waiting["text"] in {"status:waiting", f"status:changed:{expected_version}"},
|
||||
"condition read uses app text",
|
||||
)
|
||||
await call(
|
||||
"wait_until",
|
||||
selector={"content_desc": "fixture.status"},
|
||||
state="stable",
|
||||
stable_ms=150,
|
||||
timeout_ms=1000,
|
||||
poll_interval_ms=50,
|
||||
)
|
||||
|
||||
print(f"fixture iteration {iteration + 1}/{repeats}: synchronized touch", flush=True)
|
||||
await control("fixture.touch_reset", "click")
|
||||
touch = await read("fixture.touch")
|
||||
x1, y1, x2, y2 = touch["bounds"]
|
||||
left, right = max(0, x1) + 20, min(width - 1, x2) - 20
|
||||
top, bottom = max(0, y1) + 20, min(height - 70, y2) - 20
|
||||
check(right - left > 200 and bottom - top > 160, "touch fixture has room for gesture")
|
||||
center = [(left + right) // 2, (top + bottom) // 2]
|
||||
await call(
|
||||
"pinch",
|
||||
center=center,
|
||||
start_span=80,
|
||||
end_span=160,
|
||||
duration_ms=400,
|
||||
coordinate_space="pixel",
|
||||
target_description="fixture touch surface",
|
||||
)
|
||||
touch_state = state()["touch"]
|
||||
check(
|
||||
touch_state["max_pointer_count"] == 2
|
||||
and touch_state["down_count"] == touch_state["up_count"] == 1,
|
||||
"pinch has one contact with two pointers",
|
||||
)
|
||||
check(
|
||||
touch_state["cancel_count"] == 0 and 1.85 < touch_state["scale"] < 2.15,
|
||||
"pinch ratio near two without cancellation",
|
||||
)
|
||||
await control("fixture.touch_reset", "click")
|
||||
await call(
|
||||
"pinch",
|
||||
center=center,
|
||||
start_span=120,
|
||||
end_span=120,
|
||||
rotation_degrees=60,
|
||||
duration_ms=500,
|
||||
coordinate_space="pixel",
|
||||
target_description="fixture rotation",
|
||||
)
|
||||
touch_state = state()["touch"]
|
||||
check(
|
||||
abs(touch_state["rotation_degrees"] - 60) < 3,
|
||||
"two finger rotation reaches 60 degrees",
|
||||
)
|
||||
check(
|
||||
touch_state["down_count"] == touch_state["up_count"] == 1
|
||||
and touch_state["cancel_count"] == 0,
|
||||
"rotation keeps contact across segments",
|
||||
)
|
||||
await control("fixture.touch_reset", "click")
|
||||
cx, cy = center
|
||||
await call(
|
||||
"touch_gesture",
|
||||
frames=[
|
||||
{"at_ms": 0, "points": [[cx - 60, cy]]},
|
||||
{"at_ms": 250, "points": [[cx - 60, cy]]},
|
||||
{"at_ms": 500, "points": [[cx, cy - 30]]},
|
||||
{"at_ms": 750, "points": [[cx + 60, cy + 30]]},
|
||||
],
|
||||
coordinate_space="pixel",
|
||||
target_description="fixture hold then path",
|
||||
)
|
||||
touch_state = state()["touch"]
|
||||
check(
|
||||
touch_state["down_count"] == touch_state["up_count"] == 1
|
||||
and touch_state["move_count"] > 0
|
||||
and touch_state["cancel_count"] == 0,
|
||||
"multi segment hold is a single continuous contact",
|
||||
)
|
||||
events = touch_state["events"]
|
||||
down = next(event for event in events if event["action"] == 0)
|
||||
up = next(event for event in reversed(events) if event["action"] == 1)
|
||||
first = down["pointers"][0]
|
||||
moved = next(
|
||||
event
|
||||
for event in events
|
||||
if event["action"] == 2
|
||||
and any(abs(event["pointers"][0][axis] - first[axis]) > 1 for axis in ("x", "y"))
|
||||
)
|
||||
check(
|
||||
moved["time_ms"] - down["time_ms"] >= 235, "APP timestamps prove initial 250ms hold"
|
||||
)
|
||||
check(
|
||||
up["time_ms"] - down["time_ms"] >= 730,
|
||||
"APP timestamps prove total 750ms path duration",
|
||||
)
|
||||
records.append({"iteration": iteration + 1, "fixture_state": state()})
|
||||
|
||||
print("fixture scoped scroll repeated twice", flush=True)
|
||||
for _ in range(2):
|
||||
await call(
|
||||
"scroll_to",
|
||||
container={"content_desc": "fixture.scroll"},
|
||||
target={"content_desc": "fixture.bottom"},
|
||||
max_scrolls=15,
|
||||
timeout_ms=20000,
|
||||
settle_ms=200,
|
||||
)
|
||||
check((await read("fixture.bottom"))["visible"], "scoped scroll found bottom")
|
||||
await call(
|
||||
"scroll_to",
|
||||
container={"content_desc": "fixture.scroll"},
|
||||
target={"content_desc": "fixture.title"},
|
||||
direction="backward",
|
||||
max_scrolls=15,
|
||||
timeout_ms=20000,
|
||||
settle_ms=200,
|
||||
)
|
||||
check((await read("fixture.title"))["visible"], "scoped scroll restored top")
|
||||
await control("fixture.editor", "set_text", text=original["text"])
|
||||
await control("fixture.range", "set_progress", value=original["progress"])
|
||||
final_screen = await asyncio.to_thread(client.get_screen_data)
|
||||
(output / "fixture-final.jpg").write_bytes(base64.b64decode(final_screen.base64))
|
||||
print(f"PASS {len(checks)} checks / {repeats} repeated fixture iterations", flush=True)
|
||||
finally:
|
||||
if client.session is not None:
|
||||
records.append({"final_fixture_state": state()})
|
||||
(output / "result.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"checks": checks,
|
||||
"records": records,
|
||||
"elapsed_seconds": time.monotonic() - started,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
client.disconnect()
|
||||
lease.release()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--serial", required=True)
|
||||
parser.add_argument("--output", type=Path, required=True)
|
||||
parser.add_argument("--repeats", type=int, choices=range(1, 11), default=5)
|
||||
options = parser.parse_args()
|
||||
asyncio.run(verify(options.serial, options.output, options.repeats))
|
||||
@@ -27,6 +27,27 @@ async def test_operator_shell_tools():
|
||||
"swipe": {"gesture": "up"},
|
||||
"drag": {"start": [100, 200], "end": [100, 220], "target_description": "timeline"},
|
||||
"action_sequence": {"steps": [{"action": "wait", "time_in_ms": 50}]},
|
||||
"pinch": {
|
||||
"center": [500, 500],
|
||||
"start_span": 100,
|
||||
"end_span": 200,
|
||||
"target_description": "canvas",
|
||||
},
|
||||
"touch_gesture": {
|
||||
"frames": [
|
||||
{"at_ms": 0, "points": [[400, 500]]},
|
||||
{"at_ms": 500, "points": [[600, 500]]},
|
||||
],
|
||||
"target_description": "canvas path",
|
||||
},
|
||||
"read_control": {"selector": {"content_desc": "slider"}},
|
||||
"control_action": {
|
||||
"selector": {"content_desc": "slider"},
|
||||
"action": "set_progress",
|
||||
"value": 50,
|
||||
},
|
||||
"wait_until": {"selector": {"content_desc": "status"}, "state": "visible"},
|
||||
"scroll_to": {"container": {"content_desc": "list"}, "target": {"text": "target"}},
|
||||
"press_key": {"key": "ENTER"},
|
||||
"manage_app": {"action": "launch", "app_name": "Settings"},
|
||||
"wait_for_delay": {"time_in_ms": 1000},
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
"""Guard the prompt evidence contract; no model or device is invoked here."""
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.agents.checker.checker import (
|
||||
CheckReport,
|
||||
assemble_checker_prompt_segments,
|
||||
run_final_check,
|
||||
)
|
||||
from artemis.context import ArtemisContext, ExecutionSetup
|
||||
|
||||
|
||||
@pytest.mark.parametrize("entry", ["checkpoint", "final"])
|
||||
def test_bound_persistent_readout_can_supply_precision_beyond_pixel_ticks(entry):
|
||||
prompt = assemble_checker_prompt_segments(entry, [], probe_tool_registered=False)
|
||||
|
||||
# Native origin alone is insufficient: the rule must bind the displayed value
|
||||
# to the selected state and require stability plus the visible pointer check.
|
||||
for requirement in (
|
||||
"标签、节点角色/层级",
|
||||
"目标控件当前所选值或固定指针所选时间",
|
||||
"不能仅因原生节点含有数值或执行者称其可信",
|
||||
"在被验收状态中持续显示",
|
||||
"引用已停稳或暂停的证据",
|
||||
"与同一控件的指针及可辨刻度一致",
|
||||
"结合读数精度与用户容差",
|
||||
"粗刻度无法独立区分秒数，不等于",
|
||||
"无需再由粗像素刻度独立插值证明同一精确值",
|
||||
):
|
||||
assert requirement in prompt
|
||||
|
||||
assert "拖动浮层或原生时间读数与固定指针刻度冲突时" not in prompt
|
||||
|
||||
|
||||
@pytest.mark.parametrize("entry", ["checkpoint", "final"])
|
||||
def test_native_origin_does_not_promote_unbound_or_conflicting_evidence(entry):
|
||||
prompt = assemble_checker_prompt_segments(entry, [], probe_tool_registered=False)
|
||||
|
||||
for excluded_source in ("事件卡片时间", "视频画面水印", "录制起止时间", "拖动/松手浮层"):
|
||||
assert excluded_source in prompt
|
||||
assert "即使它们也能通过原生节点读取，仍须先证明上述绑定与持久性" in prompt
|
||||
assert "可辨精度上存在实际冲突" in prompt
|
||||
assert "读数仍在变化、绑定不明或无法确认已停稳/暂停" in prompt
|
||||
assert "应判证据不足（inconclusive）" in prompt
|
||||
assert "不得挑选有利读数或忽略矛盾" in prompt
|
||||
assert "可信读数明确超出用户容差时仍按通用规则判定" in prompt
|
||||
|
||||
|
||||
@pytest.mark.parametrize("entry", ["checkpoint", "final"])
|
||||
def test_pixel_fallback_remains_quantitative_and_requires_post_action_evidence(entry):
|
||||
prompt = assemble_checker_prompt_segments(entry, [], probe_tool_registered=False)
|
||||
fallback = prompt.split("没有可信持久读数时，", 1)[1]
|
||||
|
||||
assert "动作后已停稳截图的两处真实数值刻度与固定指针位置作线性插值" in fallback
|
||||
assert "单位换算、估读不确定性及目标误差" in fallback
|
||||
assert "目标误差加估读不确定性不超过用户容差" in fallback
|
||||
assert "刻度非线性或无法确定已停稳，仍应判证据不足" in fallback
|
||||
assert "操作前的校准计划及回执误差不能替代动作后测量" in fallback
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_final_review_receives_live_readout_and_rules_separately_from_report_claims():
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
ctx.execution_setup = ExecutionSetup()
|
||||
ctx.data_engine = MagicMock()
|
||||
ctx.data_engine.base_dir = "unused"
|
||||
ctx.data_engine.get_agent_friendly_steps.return_value = []
|
||||
ctx.data_engine.get_step_number.return_value = 7
|
||||
elements = "[1] 当前所选值: 23.5; [2] 已暂停"
|
||||
review = AsyncMock(return_value=CheckReport(verdicts=[]))
|
||||
|
||||
with (
|
||||
patch("artemis.agents.checker.checker._run_check_loop", new=review),
|
||||
patch(
|
||||
"artemis.agents.checker.checker._capture_final_screen",
|
||||
new=AsyncMock(return_value=("captured-image", elements)),
|
||||
),
|
||||
):
|
||||
await run_final_check(
|
||||
ctx,
|
||||
goal="核对当前所选值及容差",
|
||||
plan_text="",
|
||||
ledger=[],
|
||||
check_items=[],
|
||||
execution_report="执行者声称当前所选值为 88",
|
||||
)
|
||||
|
||||
review.assert_awaited_once()
|
||||
messages = review.await_args.args[1]
|
||||
assert "无需再由粗像素刻度独立插值证明同一精确值" in messages[0].content
|
||||
assert "不得挑选有利读数或忽略矛盾" in messages[0].content
|
||||
content = messages[1].content
|
||||
text = content[0]["text"]
|
||||
assert "Executor's Final Report (claims to verify, not independent evidence)" in text
|
||||
assert "执行者声称当前所选值为 88" in text
|
||||
assert f"# Final Screen Elements\n{elements}" in text
|
||||
assert any(
|
||||
block.get("type") == "image_url"
|
||||
and block["image_url"]["url"] == "data:image/jpeg;base64,captured-image"
|
||||
for block in content
|
||||
)
|
||||
@@ -0,0 +1,194 @@
|
||||
"""Actual action-result/history/transcript rendering must keep native evidence."""
|
||||
|
||||
from copy import deepcopy
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
|
||||
import pytest
|
||||
|
||||
from artemis.agents.validator.action_execution import exec_action
|
||||
from artemis.mcp.action_types import ActionCode, ActionResult
|
||||
from artemis.memory.transcript import EXECUTION_RESULT_MARKER, TranscriptLedger
|
||||
from artemis.utils.task_tree import format_result_clean
|
||||
|
||||
|
||||
NATIVE_ACTIONS = (
|
||||
"pinch",
|
||||
"touch_gesture",
|
||||
"read_control",
|
||||
"control_action",
|
||||
"wait_until",
|
||||
"scroll_to",
|
||||
)
|
||||
TOKEN = "a" * 64
|
||||
SELECTOR = {"resource_id": "fixture:id/editor"}
|
||||
|
||||
|
||||
def action_arguments(action):
|
||||
drag = {
|
||||
"start": [191, 501],
|
||||
"end": [192, 532],
|
||||
"coordinate_space": "pixel",
|
||||
"target_description": "observed canvas",
|
||||
}
|
||||
return {
|
||||
"pinch": {
|
||||
"center": [200, 400],
|
||||
"start_span": 100,
|
||||
"end_span": 200,
|
||||
"coordinate_space": "pixel",
|
||||
"target_description": "observed canvas",
|
||||
},
|
||||
"touch_gesture": {
|
||||
"frames": [
|
||||
{"at_ms": 0, "points": [[191, 501]]},
|
||||
{"at_ms": 400, "points": [[192, 532]]},
|
||||
],
|
||||
"coordinate_space": "pixel",
|
||||
"target_description": "observed canvas",
|
||||
},
|
||||
"read_control": {"selector": SELECTOR},
|
||||
"control_action": {
|
||||
"selector": SELECTOR,
|
||||
"action": "set_selection",
|
||||
"selection_start": 3,
|
||||
"selection_end": 5,
|
||||
},
|
||||
"wait_until": {"selector": SELECTOR, "state": "value", "expected_value": 37.5},
|
||||
"scroll_to": {"container": {"resource_id": "fixture:id/list"}, "target": SELECTOR},
|
||||
"drag": drag,
|
||||
"action_sequence": {"steps": [{"action": "drag", **drag}]},
|
||||
}[action]
|
||||
|
||||
|
||||
def control_observation():
|
||||
return {
|
||||
"resource_id": "fixture:id/editor",
|
||||
"text": " A😀B \n",
|
||||
"selection_start": 3,
|
||||
"selection_end": 5,
|
||||
"range": {"min": 0, "max": 100, "current": 37.5, "type": 1},
|
||||
"token": TOKEN,
|
||||
"content_signature": "b" * 64,
|
||||
"editable": True,
|
||||
"password": False,
|
||||
}
|
||||
|
||||
|
||||
def decode_receipt(text, label):
|
||||
marker = f"{label} receipt: "
|
||||
assert marker in text
|
||||
return json.loads(text.split(marker, 1)[1])
|
||||
|
||||
|
||||
async def render_result_pipeline(action, metadata, failed):
|
||||
message = (
|
||||
"Native reply is uncertain; observe before continuing." if failed else "Input accepted."
|
||||
)
|
||||
result = (
|
||||
ActionResult.failure(action, message, code=ActionCode.TIMEOUT, metadata=metadata)
|
||||
if failed
|
||||
else ActionResult.success(action, message, metadata=metadata)
|
||||
)
|
||||
session = SimpleNamespace(call=AsyncMock(return_value=result))
|
||||
context = SimpleNamespace(
|
||||
data_engine=None, device=SimpleNamespace(device_width=400, device_height=800)
|
||||
)
|
||||
args = action_arguments(action)
|
||||
item = {"action": action, "args": args}
|
||||
ok, error = await exec_action(context, session, item)
|
||||
assert ok is not failed
|
||||
session.call.assert_awaited_once_with(action, args)
|
||||
assert item["result"] == result.agent_message
|
||||
report = {
|
||||
"status": "failed" if failed else "dispatched",
|
||||
"execution": [{**item, "attempts": [f"Error: {error}" if failed else "Dispatched"]}],
|
||||
}
|
||||
history = format_result_clean(report)
|
||||
assert isinstance(history, str)
|
||||
|
||||
ledger = TranscriptLedger(clock=lambda: 0.0)
|
||||
ledger.stage_turn(
|
||||
[
|
||||
HumanMessage(content="Observed native controls"),
|
||||
AIMessage(content="", tool_calls=[{"name": action, "args": {}, "id": "native-call"}]),
|
||||
ToolMessage(content="Action Recorded", tool_call_id="native-call"),
|
||||
]
|
||||
)
|
||||
ledger.commit_staged(step_key="native-step", validator_result=report)
|
||||
rendered = ledger.render([HumanMessage(content="Next observation")])
|
||||
execution_results = [
|
||||
message.content[0]["text"]
|
||||
for message in rendered
|
||||
if isinstance(message, HumanMessage)
|
||||
and isinstance(message.content, list)
|
||||
and message.content
|
||||
and EXECUTION_RESULT_MARKER in message.content[0].get("text", "")
|
||||
]
|
||||
assert len(execution_results) == 1
|
||||
transcript = execution_results[0]
|
||||
assert f"Status: {report['status']}" in transcript
|
||||
if failed:
|
||||
assert "Error:" in history and "Error:" in transcript
|
||||
return result.agent_message, history, transcript
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("action", NATIVE_ACTIONS)
|
||||
@pytest.mark.parametrize("failed", [False, True])
|
||||
async def test_every_native_action_keeps_readings_selection_range_and_token_in_actual_result_rendering(
|
||||
action, failed
|
||||
):
|
||||
node = control_observation()
|
||||
read_only = action in {"read_control", "wait_until"}
|
||||
execution = (
|
||||
("not_started" if read_only else "unknown")
|
||||
if failed
|
||||
else ("observed" if read_only else "completed")
|
||||
)
|
||||
# The renderer must retain optional observed evidence even on gesture receipts.
|
||||
metadata = {
|
||||
"action": action,
|
||||
"execution": execution,
|
||||
"node": deepcopy(node),
|
||||
"native_receipt": {
|
||||
"success": not failed,
|
||||
"execution": execution,
|
||||
"code": "TIMEOUT" if failed else "OK",
|
||||
"node": deepcopy(node),
|
||||
},
|
||||
"verification": "pending_observation",
|
||||
}
|
||||
stages = await render_result_pipeline(action, metadata, failed)
|
||||
for rendered in stages:
|
||||
recovered = decode_receipt(rendered, "Native")
|
||||
assert recovered == metadata
|
||||
assert recovered["node"]["text"] == " A😀B \n"
|
||||
assert recovered["node"]["range"]["current"] == 37.5
|
||||
assert (recovered["node"]["selection_start"], recovered["node"]["selection_end"]) == (3, 5)
|
||||
assert recovered["node"]["token"] == TOKEN
|
||||
assert recovered["native_receipt"]["success"] is not failed
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("action", ["drag", "action_sequence"])
|
||||
@pytest.mark.parametrize("failed", [False, True])
|
||||
async def test_existing_drag_and_sequence_geometry_and_partial_receipts_still_render(
|
||||
action, failed
|
||||
):
|
||||
metadata = {
|
||||
"execution": "partial" if failed else "completed",
|
||||
"coordinate_space": "pixel",
|
||||
"start": [191, 501],
|
||||
"end": [192, 532],
|
||||
"calibration": {"axis": "y", "target_value": 37.5},
|
||||
}
|
||||
if action == "action_sequence":
|
||||
metadata["sequence_results"] = [
|
||||
{"action": "click", "execution": "completed", "completed_taps": 1},
|
||||
{"action": "drag", "execution": "unknown" if failed else "completed"},
|
||||
]
|
||||
for rendered in await render_result_pipeline(action, metadata, failed):
|
||||
assert decode_receipt(rendered, "Gesture") == metadata
|
||||
@@ -35,6 +35,7 @@ from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, Tool
|
||||
from artemis.agents.operator.operator import OperatorNode
|
||||
from artemis.agents.operator.prompts import load_operator_prompts
|
||||
from artemis.agents.operator.prompts import (
|
||||
OPERATOR_PROMPT_TOOLSET,
|
||||
PLAN_HISTORY_STATIC_POINTER,
|
||||
PLAN_HISTORY_TEMPLATE_SECTION,
|
||||
PromptBuilder,
|
||||
@@ -54,8 +55,8 @@ from artemis.memory.transcript import (
|
||||
|
||||
# SHA-256 snapshots of the legacy system message with the fixed inputs below.
|
||||
# Update these when an intentional template change alters the rendered prompt.
|
||||
GOLDEN_EMPTY_PLAN = "23c3d6b5fc3694275e7015f7bd0901bde54aae54fec7a8c68d4f8529c65ef4bc"
|
||||
GOLDEN_SENTINEL_PLAN = "370c53f1d088a4120c63fbaea6d895f121c683b8d3c8cc7d61105d6e1d223128"
|
||||
GOLDEN_EMPTY_PLAN = "f5ee50f4923fed4cb943cb80087d39ffffe66f259cfae4a54f27a244d7b06453"
|
||||
GOLDEN_SENTINEL_PLAN = "bb725e2d096f6aa0cf1773809b2d0e672475589782b9c72348d9eaf8481230b9"
|
||||
|
||||
SCREENSHOT_B64 = base64.b64encode(b"fake-jpeg-bytes").decode("utf-8")
|
||||
|
||||
@@ -84,6 +85,42 @@ async def test_legacy_prompt_bytes_match_chinese_prompt_golden():
|
||||
assert hashlib.sha256(sentinel.encode("utf-8")).hexdigest() == GOLDEN_SENTINEL_PLAN
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("transcript", [False, True])
|
||||
async def test_legacy_and_transcript_prompts_omit_all_unavailable_native_capabilities(transcript):
|
||||
native_tools = {
|
||||
"pinch",
|
||||
"touch_gesture",
|
||||
"read_control",
|
||||
"control_action",
|
||||
"wait_until",
|
||||
"scroll_to",
|
||||
}
|
||||
available_tools = OPERATOR_PROMPT_TOOLSET - native_tools
|
||||
ctx = SimpleNamespace(
|
||||
execution_setup=None,
|
||||
actuator=SimpleNamespace(capabilities=lambda: available_tools),
|
||||
data_engine=None,
|
||||
)
|
||||
state = SimpleNamespace(initial_goal="FILTERED_FIXED_GOAL")
|
||||
if transcript:
|
||||
system = render_transcript_static_system(load_operator_prompts(), ctx, state)
|
||||
else:
|
||||
builder = PromptBuilder()
|
||||
await TemplatePromptComponent()(
|
||||
builder,
|
||||
state,
|
||||
ctx,
|
||||
prompts=load_operator_prompts(),
|
||||
template_name="main_template",
|
||||
)
|
||||
system = "".join(builder.system_parts)
|
||||
assert "FILTERED_FIXED_GOAL" in system
|
||||
assert "`click`" in system and "`drag`" in system
|
||||
for tool in native_tools:
|
||||
assert tool not in system
|
||||
|
||||
|
||||
def test_main_template_contains_exactly_one_plan_history_section():
|
||||
template = load_operator_prompts()["main_template"]
|
||||
assert template.count(PLAN_HISTORY_TEMPLATE_SECTION) == 1
|
||||
|
||||
@@ -38,6 +38,14 @@ OPERATOR_DEVICE_TOOLS = (
|
||||
"wait_for_delay",
|
||||
"long_press",
|
||||
)
|
||||
OPERATOR_NATIVE_TOOLS = (
|
||||
"pinch",
|
||||
"touch_gesture",
|
||||
"read_control",
|
||||
"control_action",
|
||||
"wait_until",
|
||||
"scroll_to",
|
||||
)
|
||||
|
||||
|
||||
def _tool_refs(text: str, tool: str) -> list[str]:
|
||||
@@ -88,6 +96,7 @@ def test_full_toolset_is_the_default():
|
||||
@pytest.mark.parametrize(
|
||||
"tool",
|
||||
OPERATOR_DEVICE_TOOLS
|
||||
+ OPERATOR_NATIVE_TOOLS
|
||||
+ ("video_analyzer", "search_history", "replay_steps", "get_step_screenshot"),
|
||||
)
|
||||
def test_removed_tool_leaves_no_reference(tool):
|
||||
@@ -99,6 +108,23 @@ def test_removed_tool_leaves_no_reference(tool):
|
||||
assert not _tool_refs(out, tool), f"'{tool}' still referenced in {name}"
|
||||
|
||||
|
||||
def test_removing_all_native_capabilities_leaves_no_native_names_in_any_template():
|
||||
available_tools = OPERATOR_PROMPT_TOOLSET - set(OPERATOR_NATIVE_TOOLS)
|
||||
for name, template in load_operator_prompts().items():
|
||||
out = apply_operator_prompt_contract(template, available_tools=available_tools)
|
||||
for tool in OPERATOR_NATIVE_TOOLS:
|
||||
assert tool not in out, f"'{tool}' still advertised without native capability in {name}"
|
||||
assert "`click`" in out and "`drag`" in out
|
||||
assert (
|
||||
"Physical device actions (`click`, `action_sequence`, `drag`, `input_text`, `swipe`,"
|
||||
" `press_key`, `manage_app`, `wait_for_delay`, `long_press`)" in out
|
||||
)
|
||||
assert (
|
||||
"Turn-Ending Action (`click`, `action_sequence`, `drag`, `swipe`, `input_text`,"
|
||||
" `long_press`, `press_key`, `manage_app`, or `wait_for_delay`)" in out
|
||||
)
|
||||
|
||||
|
||||
def test_removed_swipe_takes_its_teaching_along():
|
||||
"""Gating removes the drag/slider teaching, not just the backticked name."""
|
||||
template = load_operator_prompts()["main_template"]
|
||||
@@ -131,26 +157,30 @@ def test_reduced_enums_stay_well_formed():
|
||||
available_tools=OPERATOR_PROMPT_TOOLSET - {"manage_app", "wait_for_delay"},
|
||||
)
|
||||
assert (
|
||||
"Physical device actions (`click`, `action_sequence`, `drag`, `input_text`, `swipe`, `press_key`, `long_press`)"
|
||||
in out
|
||||
"Physical device actions (`click`, `action_sequence`, `drag`, `pinch`, `touch_gesture`,"
|
||||
" `read_control`, `control_action`, `wait_until`, `scroll_to`, `input_text`, `swipe`,"
|
||||
" `press_key`, `long_press`)" in out
|
||||
)
|
||||
assert (
|
||||
"Turn-Ending Action (`click`, `action_sequence`, `drag`, `swipe`, `input_text`, `long_press`, or `press_key`)"
|
||||
in out
|
||||
"Turn-Ending Action (`click`, `action_sequence`, `drag`, `pinch`, `touch_gesture`,"
|
||||
" `read_control`, `control_action`, `wait_until`, `scroll_to`, `swipe`, `input_text`,"
|
||||
" `long_press`, or `press_key`)" in out
|
||||
)
|
||||
|
||||
|
||||
def test_full_set_enum_slots_render_verbatim():
|
||||
"""With every tool present the historical enumeration wording is reproduced."""
|
||||
"""Every supported tool appears in the exact physical and turn-ending order."""
|
||||
template = load_operator_prompts()["main_template"]
|
||||
out = apply_operator_prompt_contract(template)
|
||||
assert (
|
||||
"Physical device actions (`click`, `action_sequence`, `drag`, `input_text`, `swipe`, `press_key`,"
|
||||
" `manage_app`, `wait_for_delay`, `long_press`)" in out
|
||||
"Physical device actions (`click`, `action_sequence`, `drag`, `pinch`, `touch_gesture`,"
|
||||
" `read_control`, `control_action`, `wait_until`, `scroll_to`, `input_text`, `swipe`,"
|
||||
" `press_key`, `manage_app`, `wait_for_delay`, `long_press`)" in out
|
||||
)
|
||||
assert (
|
||||
"Turn-Ending Action (`click`, `action_sequence`, `drag`, `swipe`, `input_text`, `long_press`,"
|
||||
" `press_key`, `manage_app`, or `wait_for_delay`)" in out
|
||||
"Turn-Ending Action (`click`, `action_sequence`, `drag`, `pinch`, `touch_gesture`,"
|
||||
" `read_control`, `control_action`, `wait_until`, `scroll_to`, `swipe`, `input_text`,"
|
||||
" `long_press`, `press_key`, `manage_app`, or `wait_for_delay`)" in out
|
||||
)
|
||||
assert "Helper/Subagent tools (`ask_explorer`, `ask_diagnoser`, `video_analyzer`)" in out
|
||||
assert "ADB/task tools (`run_adb_command`, `manage_task`, `android_file`)" in out
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
"""Native input is one-shot and remains owned through uncertain transport cleanup."""
|
||||
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
import urllib.error
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.clients import accessibility_client as module
|
||||
from artemis.clients.accessibility_client import AccessibilityClient
|
||||
from artemis.clients.screen_client_factory import FallbackScreenClient
|
||||
from artemis.runtime.helper_manager import HelperSession
|
||||
from artemis.runtime.native_protocol import input_completion_seconds
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def native(monkeypatch):
|
||||
manager = Mock()
|
||||
client = AccessibilityClient("test-device", manager=manager)
|
||||
client._session = HelperSession(
|
||||
serial="test-device",
|
||||
local_port=40001,
|
||||
transport_id="1",
|
||||
version_code=12,
|
||||
version_name="1.3.0",
|
||||
owns_forward=True,
|
||||
token="test-only",
|
||||
protocol_version=3,
|
||||
capabilities=("inspect_control", "control_action", "touch_gesture", "cancel_gesture"),
|
||||
)
|
||||
clock = [0.0]
|
||||
sleeps = []
|
||||
|
||||
def sleep(seconds):
|
||||
sleeps.append(seconds)
|
||||
clock[0] += seconds
|
||||
|
||||
monkeypatch.setattr(module, "time", SimpleNamespace(monotonic=lambda: clock[0], sleep=sleep))
|
||||
client._http_once = Mock(return_value=b'{"success":true,"execution":"completed","code":"OK"}')
|
||||
return client, manager, clock, sleeps
|
||||
|
||||
|
||||
def test_exactly_one_request_and_command_cannot_be_overridden(native):
|
||||
client, manager, _, sleeps = native
|
||||
receipt = {
|
||||
"success": True,
|
||||
"execution": "observed",
|
||||
"code": "OK",
|
||||
"node": {"selection_start": 3},
|
||||
}
|
||||
client._http_once.return_value = json.dumps(receipt).encode()
|
||||
assert client.native_request("inspect_control", {"cmd": "swipe"}, timeout=0.02) == receipt
|
||||
client._http_once.assert_called_once_with(
|
||||
client.session, "/action", {"cmd": "inspect_control"}, 0.02
|
||||
)
|
||||
assert sleeps == []
|
||||
manager.attach.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"status,execution,attempts,repairs",
|
||||
[
|
||||
(400, "not_started", 1, 0),
|
||||
(401, "not_started", 1, 0),
|
||||
(403, "not_started", 1, 0),
|
||||
(503, "unknown", 1, 0),
|
||||
],
|
||||
)
|
||||
def test_http_rejection_never_replays_native_input(native, status, execution, attempts, repairs):
|
||||
client, manager, _, sleeps = native
|
||||
client._http_once.side_effect = urllib.error.HTTPError(
|
||||
"http://test", status, "rejected", {}, None
|
||||
)
|
||||
result = client.native_request("control_action", {"action": "click"}, timeout=0.02)
|
||||
assert result["execution"] == execution and not result["success"]
|
||||
assert client._http_once.call_count == attempts
|
||||
assert client._http_once.call_args.args[-1] >= 5
|
||||
assert manager.push_token.call_count == repairs
|
||||
manager.reattach.assert_not_called()
|
||||
assert bool(sleeps) == (execution == "unknown")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"response", [b"broken", b"[]", b"{}", b'{"success":true,"execution":"unknown"}']
|
||||
)
|
||||
def test_uncertain_write_drains_without_retry(native, response):
|
||||
client, _, clock, sleeps = native
|
||||
client._http_once.return_value = response
|
||||
result = client.native_request("control_action", {"action": "click"})
|
||||
assert result["execution"] == "unknown"
|
||||
assert sleeps == [5.0] and clock[0] == 5.0
|
||||
client._http_once.assert_called_once()
|
||||
|
||||
|
||||
def test_partial_transport_elapsed_time_counts_toward_input_drain(native):
|
||||
client, _, clock, sleeps = native
|
||||
|
||||
def broken(*args):
|
||||
clock[0] = 2
|
||||
raise OSError("response lost after dispatch")
|
||||
|
||||
client._http_once.side_effect = broken
|
||||
assert client.native_request("control_action", {})["execution"] == "unknown"
|
||||
assert sleeps == [3.0] and clock[0] == 5
|
||||
client._http_once.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("command", ["inspect_control", "cancel_gesture"])
|
||||
def test_non_input_calls_do_not_drain_on_failure(native, command):
|
||||
client, _, _, sleeps = native
|
||||
client._http_once.side_effect = OSError("offline")
|
||||
result = client.native_request(command, {}, timeout=0.02)
|
||||
assert not result["success"] and sleeps == []
|
||||
client._http_once.assert_called_once()
|
||||
|
||||
|
||||
def test_no_session_or_capability_cannot_trigger_attach_or_input(native):
|
||||
client, manager, _, sleeps = native
|
||||
client._session.capabilities = ()
|
||||
assert client.native_request("touch_gesture", {})["code"] == "UNSUPPORTED"
|
||||
client._session = None
|
||||
assert client.native_capabilities == frozenset()
|
||||
assert client.native_request("control_action", {})["execution"] == "not_started"
|
||||
client._http_once.assert_not_called()
|
||||
manager.attach.assert_not_called()
|
||||
assert sleeps == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"duration,budget", [(0, 5), (1000, 6), (10000, 15), (999999, 15), (None, 15)]
|
||||
)
|
||||
def test_touch_completion_budget_is_bounded(duration, budget):
|
||||
assert input_completion_seconds("touch_gesture", {"frames": [{"at_ms": duration}]}) == budget
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend", ["helper", "uiautomator"])
|
||||
def test_native_input_never_uses_fallback_dispatch(backend):
|
||||
client = object.__new__(FallbackScreenClient)
|
||||
client._active_backend = backend
|
||||
client._helper = Mock()
|
||||
client._helper.native_capabilities = frozenset({"control_action"})
|
||||
unknown = {"success": False, "execution": "unknown", "code": "TRANSPORT_ERROR"}
|
||||
client._helper.native_request.return_value = unknown
|
||||
client._call = Mock(side_effect=AssertionError("fallback is forbidden"))
|
||||
result = client.native_request("control_action", {})
|
||||
if backend == "helper":
|
||||
assert result is unknown and client.native_capabilities == frozenset({"control_action"})
|
||||
client._helper.native_request.assert_called_once()
|
||||
else:
|
||||
assert result["execution"] == "not_started" and client.native_capabilities == frozenset()
|
||||
client._helper.native_request.assert_not_called()
|
||||
client._call.assert_not_called()
|
||||
@@ -0,0 +1,70 @@
|
||||
"""Cancellation drains input threads, while read-only work cannot hold the deadline."""
|
||||
|
||||
import asyncio
|
||||
from threading import Event
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.drivers.android.adb_driver import AndroidAdbDriver
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("command", ["control_action", "touch_gesture", "inspect_control"])
|
||||
async def test_cancel_keeps_input_owned_until_transport_finishes(command):
|
||||
started, release, finished, cancel_seen = Event(), Event(), Event(), Event()
|
||||
|
||||
def request(name, args, *, timeout):
|
||||
if name == "cancel_gesture":
|
||||
cancel_seen.set()
|
||||
return {"success": True, "execution": "observed", "code": "CANCEL_RECORDED"}
|
||||
started.set()
|
||||
try:
|
||||
if not release.wait(2):
|
||||
raise RuntimeError("test transport was not released")
|
||||
return {"success": True, "execution": "completed", "code": "OK"}
|
||||
finally:
|
||||
finished.set()
|
||||
|
||||
driver = object.__new__(AndroidAdbDriver)
|
||||
driver._ui_adb_client = SimpleNamespace(native_request=request)
|
||||
operation = asyncio.create_task(driver.native_request(command, {"operation_id": "test-cancel"}))
|
||||
try:
|
||||
assert await asyncio.to_thread(started.wait, 1)
|
||||
operation.cancel()
|
||||
done, _ = await asyncio.wait({operation}, timeout=0.05)
|
||||
if command == "inspect_control":
|
||||
assert operation in done and not finished.is_set()
|
||||
else:
|
||||
assert operation not in done and not finished.is_set()
|
||||
operation.cancel()
|
||||
done, _ = await asyncio.wait({operation}, timeout=0.02)
|
||||
assert not done
|
||||
assert cancel_seen.is_set() == (command == "touch_gesture")
|
||||
finally:
|
||||
release.set()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await operation
|
||||
assert await asyncio.to_thread(finished.wait, 1)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancelled_input_transport_failure_does_not_replace_cancellation():
|
||||
started, release = Event(), Event()
|
||||
|
||||
def request(*args, **kwargs):
|
||||
started.set()
|
||||
release.wait(2)
|
||||
raise OSError("response lost")
|
||||
|
||||
driver = object.__new__(AndroidAdbDriver)
|
||||
driver._ui_adb_client = SimpleNamespace(native_request=request)
|
||||
operation = asyncio.create_task(driver.native_request("control_action", {}))
|
||||
try:
|
||||
assert await asyncio.to_thread(started.wait, 1)
|
||||
operation.cancel()
|
||||
await asyncio.sleep(0)
|
||||
finally:
|
||||
release.set()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await operation
|
||||
@@ -85,6 +85,12 @@ def test_tool_declarations_match_fixture():
|
||||
"SWIPE_TOOL": "swipe",
|
||||
"DRAG_TOOL": "drag",
|
||||
"ACTION_SEQUENCE_TOOL": "action_sequence",
|
||||
"PINCH_TOOL": "pinch",
|
||||
"TOUCH_GESTURE_TOOL": "touch_gesture",
|
||||
"READ_CONTROL_TOOL": "read_control",
|
||||
"CONTROL_ACTION_TOOL": "control_action",
|
||||
"WAIT_UNTIL_TOOL": "wait_until",
|
||||
"SCROLL_TO_TOOL": "scroll_to",
|
||||
"PRESS_KEY_TOOL": "press_key",
|
||||
"MANAGE_APP_TOOL": "manage_app",
|
||||
"WAIT_FOR_DELAY_TOOL": "wait_for_delay",
|
||||
@@ -125,6 +131,12 @@ def test_validator_declaration_order_is_stable():
|
||||
"swipe",
|
||||
"drag",
|
||||
"action_sequence",
|
||||
"pinch",
|
||||
"touch_gesture",
|
||||
"read_control",
|
||||
"control_action",
|
||||
"wait_until",
|
||||
"scroll_to",
|
||||
"press_key",
|
||||
"read_note",
|
||||
"list_notes",
|
||||
|
||||
@@ -0,0 +1,404 @@
|
||||
"""Native action input contracts and geometry, without dispatching device input."""
|
||||
|
||||
from copy import deepcopy
|
||||
import math
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from artemis.mcp.native_actions import (
|
||||
NATIVE_REQUESTS,
|
||||
ControlActionRequest,
|
||||
ControlSelector,
|
||||
PinchRequest,
|
||||
TouchGestureRequest,
|
||||
WaitUntilRequest,
|
||||
plan_pinch,
|
||||
plan_touch,
|
||||
validate_native_action,
|
||||
)
|
||||
|
||||
|
||||
def touch(**overrides):
|
||||
return {
|
||||
"frames": [
|
||||
{"at_ms": 0, "points": [[100, 200]]},
|
||||
{"at_ms": 400, "points": [[300, 600]]},
|
||||
],
|
||||
"coordinate_space": "pixel",
|
||||
"target_description": "observed canvas",
|
||||
**overrides,
|
||||
}
|
||||
|
||||
|
||||
def pinch(**overrides):
|
||||
return {
|
||||
"center": [200, 400],
|
||||
"start_span": 100,
|
||||
"end_span": 200,
|
||||
"coordinate_space": "pixel",
|
||||
"target_description": "observed canvas",
|
||||
**overrides,
|
||||
}
|
||||
|
||||
|
||||
VALID_REQUESTS = {
|
||||
"pinch": pinch(),
|
||||
"touch_gesture": touch(),
|
||||
"read_control": {"selector": {"resource_id": "fixture:id/range"}},
|
||||
"control_action": {
|
||||
"selector": {"resource_id": "fixture:id/range"},
|
||||
"action": "set_progress",
|
||||
"value": 37.5,
|
||||
},
|
||||
"wait_until": {"selector": {"text": "Ready"}, "state": "visible"},
|
||||
"scroll_to": {
|
||||
"container": {"resource_id": "fixture:id/list"},
|
||||
"target": {"text": "Destination"},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", VALID_REQUESTS)
|
||||
def test_every_native_request_validates_before_dispatch_and_forbids_extra_fields(name):
|
||||
args = deepcopy(VALID_REQUESTS[name])
|
||||
assert validate_native_action(name, args, 400, 800).model_dump(exclude_none=True)
|
||||
with pytest.raises(ValidationError):
|
||||
validate_native_action(name, {**args, "coordinate_spce": "pixel"}, 400, 800)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"selector",
|
||||
[
|
||||
{},
|
||||
{"bounds": [0, 0, 10, 10]},
|
||||
{"text": ""},
|
||||
{"text": 123},
|
||||
{"resource_id": "item", "resourceId": "typo"},
|
||||
{"resource_id": "item", "bounds": [0, 0, 10]},
|
||||
{"resource_id": "item", "bounds": [0, 0, 10, 10, 20]},
|
||||
{"resource_id": "item", "bounds": [10, 0, 10, 20]},
|
||||
{"resource_id": "item", "bounds": [20, 0, 10, 20]},
|
||||
{"resource_id": "item", "bounds": [0, 0, 10, 0]},
|
||||
{"resource_id": "item", "bounds": [False, 0, 10, 10]},
|
||||
{"resource_id": "item", "bounds": [0, 0, 10.0, 10]},
|
||||
{"resource_id": "item", "bounds": [0, 0, "10", 10]},
|
||||
],
|
||||
)
|
||||
def test_selectors_require_observed_semantics_and_strict_nonempty_bounds(selector):
|
||||
with pytest.raises(ValidationError):
|
||||
ControlSelector.model_validate(selector)
|
||||
|
||||
|
||||
def test_exact_selector_strings_keep_whitespace_and_do_not_choose_one_attribute():
|
||||
observed = {
|
||||
"resource_id": "fixture:id/editor",
|
||||
"text": " A😀B \n",
|
||||
"content_desc": " description \t",
|
||||
"class_name": "android.widget.EditText",
|
||||
"package_name": "fixture",
|
||||
"bounds": [-5, 20, 100, 60],
|
||||
}
|
||||
assert ControlSelector.model_validate(observed).model_dump(exclude_none=True) == observed
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"frames",
|
||||
[
|
||||
[{"at_ms": 0, "points": [[1, 2]]}],
|
||||
[{"at_ms": i, "points": [[1, 2]]} for i in range(33)],
|
||||
[{"at_ms": 1, "points": [[1, 2]]}, {"at_ms": 2, "points": [[2, 3]]}],
|
||||
[{"at_ms": 0, "points": [[1, 2]]}, {"at_ms": 0, "points": [[2, 3]]}],
|
||||
[{"at_ms": 0, "points": [[1, 2]]}, {"at_ms": -1, "points": [[2, 3]]}],
|
||||
[{"at_ms": 0, "points": [[1, 2]]}, {"at_ms": 10001, "points": [[2, 3]]}],
|
||||
[{"at_ms": 0, "points": []}, {"at_ms": 10, "points": []}],
|
||||
[{"at_ms": 0, "points": [[1, 2]] * 6}, {"at_ms": 10, "points": [[2, 3]] * 6}],
|
||||
[{"at_ms": 0, "points": [[1, 2]]}, {"at_ms": 10, "points": [[2, 3], [4, 5]]}],
|
||||
[{"at_ms": 0, "points": [[1]]}, {"at_ms": 10, "points": [[2, 3]]}],
|
||||
[{"at_ms": 0, "points": [[1, 2, 3]]}, {"at_ms": 10, "points": [[2, 3]]}],
|
||||
[{"at_ms": 0, "points": [[True, 2]]}, {"at_ms": 10, "points": [[2, 3]]}],
|
||||
[{"at_ms": 0, "points": [[1.0, 2]]}, {"at_ms": 10, "points": [[2, 3]]}],
|
||||
[{"at_ms": 0, "points": [["1", 2]]}, {"at_ms": 10, "points": [[2, 3]]}],
|
||||
[{"at_ms": 0, "points": [[1, 2]]}, {"at_ms": "10", "points": [[2, 3]]}],
|
||||
[{"at_ms": 0, "points": [[1, 2]]}, {"at_ms": 10.0, "points": [[2, 3]]}],
|
||||
[{"at_ms": 0, "points": [[1, 2]]}, {"at_ms": True, "points": [[2, 3]]}],
|
||||
[
|
||||
{"at_ms": 0, "points": [[1, 2]], "pressure": 0.5},
|
||||
{"at_ms": 10, "points": [[2, 3]]},
|
||||
],
|
||||
],
|
||||
)
|
||||
def test_touch_timelines_and_pointer_shapes_are_strict(frames):
|
||||
with pytest.raises(ValidationError):
|
||||
TouchGestureRequest.model_validate(touch(frames=frames))
|
||||
|
||||
|
||||
def test_pixel_keyframes_preserve_holds_and_every_intermediate_segment():
|
||||
frames = [
|
||||
{"at_ms": 0, "points": [[191, 501], [211, 502]]},
|
||||
{"at_ms": 150, "points": [[191, 501], [211, 502]]},
|
||||
{"at_ms": 900, "points": [[192, 532], [212, 533]]},
|
||||
]
|
||||
planned = plan_touch(TouchGestureRequest.model_validate(touch(frames=frames)), 400, 800)
|
||||
assert planned["frames"] == frames
|
||||
assert planned["coordinate_space"] == "pixel"
|
||||
assert planned["screen_size"] == [400, 800]
|
||||
|
||||
|
||||
def test_full_supported_pointer_count_frame_count_and_duration_remain_valid():
|
||||
frames = [
|
||||
{
|
||||
"at_ms": round(frame * 10000 / 31),
|
||||
"points": [[50 + finger * 30, 100 + frame] for finger in range(5)],
|
||||
}
|
||||
for frame in range(32)
|
||||
]
|
||||
request = validate_native_action("touch_gesture", touch(frames=frames), 400, 800)
|
||||
assert plan_touch(request, 400, 800)["frames"] == frames
|
||||
|
||||
|
||||
def test_normalized_points_use_each_display_axis_and_accept_the_nominal_edges():
|
||||
request = TouchGestureRequest.model_validate(
|
||||
touch(
|
||||
coordinate_space="normalized",
|
||||
frames=[
|
||||
{"at_ms": 0, "points": [[0, 0]]},
|
||||
{"at_ms": 100, "points": [[500, 500]]},
|
||||
{"at_ms": 200, "points": [[1000, 1000]]},
|
||||
],
|
||||
)
|
||||
)
|
||||
assert [f["points"] for f in plan_touch(request, 720, 1600)["frames"]] == [
|
||||
[[0, 0]],
|
||||
[[360, 800]],
|
||||
[[719, 1599]],
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"space,point",
|
||||
[
|
||||
("pixel", [-1, 0]),
|
||||
("pixel", [400, 0]),
|
||||
("pixel", [0, 800]),
|
||||
("normalized", [-1, 500]),
|
||||
("normalized", [1001, 500]),
|
||||
("normalized", [500, 1001]),
|
||||
],
|
||||
)
|
||||
def test_offscreen_paths_fail_before_any_coordinate_can_be_clamped(space, point):
|
||||
args = touch(
|
||||
coordinate_space=space,
|
||||
frames=[{"at_ms": 0, "points": [[100, 100]]}, {"at_ms": 100, "points": [point]}],
|
||||
)
|
||||
with pytest.raises(ValueError, match="outside"):
|
||||
validate_native_action("touch_gesture", args, 400, 800)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dimensions", [(0, 800), (400, 0), (-1, 800)])
|
||||
def test_invalid_display_dimensions_cannot_produce_a_plan(dimensions):
|
||||
with pytest.raises(ValueError, match="positive"):
|
||||
validate_native_action("touch_gesture", touch(), *dimensions)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"before,after",
|
||||
[
|
||||
([[100, 100], [300, 100]], [[300, 100], [100, 100]]),
|
||||
([[100, 100], [300, 100]], [[200, 200], [200, 200]]),
|
||||
([[100, 100], [100, 100]], [[100, 200], [300, 200]]),
|
||||
],
|
||||
)
|
||||
def test_fingers_may_not_merge_at_frames_or_between_frames(before, after):
|
||||
with pytest.raises(ValueError, match="remain distinct"):
|
||||
validate_native_action(
|
||||
"touch_gesture",
|
||||
touch(frames=[{"at_ms": 0, "points": before}, {"at_ms": 100, "points": after}]),
|
||||
400,
|
||||
800,
|
||||
)
|
||||
|
||||
|
||||
def test_paths_can_cross_geometrically_when_fingers_reach_the_crossing_at_different_times():
|
||||
args = touch(
|
||||
frames=[
|
||||
{"at_ms": 0, "points": [[0, 100], [100, 0]]},
|
||||
{"at_ms": 400, "points": [[200, 100], [100, 400]]},
|
||||
]
|
||||
)
|
||||
assert (
|
||||
plan_touch(TouchGestureRequest.model_validate(args), 400, 800)["frames"] == args["frames"]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dimensions", [(720, 1600), (1600, 720)])
|
||||
def test_normalized_pinch_spans_use_short_edge_without_stretching_diagonal_angles(dimensions):
|
||||
width, height = dimensions
|
||||
request = PinchRequest.model_validate(
|
||||
pinch(
|
||||
center=[500, 500],
|
||||
start_span=200,
|
||||
end_span=400,
|
||||
angle_degrees=45,
|
||||
coordinate_space="normalized",
|
||||
)
|
||||
)
|
||||
planned = plan_pinch(request, width, height)
|
||||
for frame, expected_span in zip(planned["frames"], [144, 288], strict=True):
|
||||
first, second = frame["points"]
|
||||
assert second[0] - first[0] == second[1] - first[1]
|
||||
assert math.dist(first, second) == pytest.approx(expected_span, abs=1.5)
|
||||
assert [(first[i] + second[i]) / 2 for i in (0, 1)] == [width / 2, height / 2]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("rotation", [-360, -180, -90, 90, 180, 360])
|
||||
def test_rotation_uses_curved_synchronized_keyframes_and_keeps_both_fingers_down(rotation):
|
||||
planned = plan_pinch(
|
||||
PinchRequest.model_validate(
|
||||
pinch(center=[500, 500], start_span=200, end_span=200, rotation_degrees=rotation)
|
||||
),
|
||||
1000,
|
||||
1000,
|
||||
)
|
||||
frames = planned["frames"]
|
||||
assert len(frames) == abs(rotation) // 15 + 1
|
||||
assert frames[0]["points"] == [[400, 500], [600, 500]]
|
||||
assert frames[0]["at_ms"] == 0 and frames[-1]["at_ms"] == 800
|
||||
assert all(a["at_ms"] < b["at_ms"] for a, b in zip(frames, frames[1:]))
|
||||
assert all(len(frame["points"]) == 2 for frame in frames)
|
||||
quarter_turn = frames[6]["points"][1]
|
||||
assert quarter_turn == [500, 600 if rotation > 0 else 400]
|
||||
for frame in frames:
|
||||
assert math.dist(*frame["points"]) == pytest.approx(200, abs=1.5)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"overrides",
|
||||
[
|
||||
{"start_span": 0},
|
||||
{"end_span": -1},
|
||||
{"start_span": True},
|
||||
{"end_span": "200"},
|
||||
{"end_span": 100},
|
||||
{"rotation_degrees": 361},
|
||||
{"angle_degrees": float("nan")},
|
||||
{"angle_degrees": float("inf")},
|
||||
{"duration_ms": 0},
|
||||
{"duration_ms": 10001},
|
||||
{"duration_ms": 1.0},
|
||||
{"duration_ms": 20, "rotation_degrees": 360},
|
||||
{"center": [0, 0]},
|
||||
{"coordinate_space": "screen"},
|
||||
],
|
||||
)
|
||||
def test_invalid_pinch_geometry_and_units_fail_during_planning(overrides):
|
||||
with pytest.raises(ValueError):
|
||||
validate_native_action("pinch", pinch(**overrides), 400, 800)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("action", ["set_text", "replace_selection"])
|
||||
@pytest.mark.parametrize("text", ["", " A😀B \n\t"])
|
||||
def test_precise_text_edits_preserve_empty_input_unicode_and_whitespace(action, text):
|
||||
request = ControlActionRequest.model_validate(
|
||||
{"selector": {"resource_id": "editor"}, "action": action, "text": text}
|
||||
)
|
||||
assert request.model_dump(mode="json", exclude_none=True)["text"] == text
|
||||
|
||||
|
||||
def test_selection_offsets_are_preserved_as_utf16_units_without_python_character_conversion():
|
||||
text = "A😀B"
|
||||
end_of_emoji = len(text[:2].encode("utf-16-le")) // 2
|
||||
request = ControlActionRequest.model_validate(
|
||||
{
|
||||
"selector": {"text": text},
|
||||
"action": "set_selection",
|
||||
"selection_start": 1,
|
||||
"selection_end": end_of_emoji,
|
||||
}
|
||||
)
|
||||
wire = request.model_dump(mode="json", exclude_none=True)
|
||||
assert (wire["selection_start"], wire["selection_end"]) == (1, 3)
|
||||
# Bounds and surrogate-pair boundaries require the fresh native text, checked by Java.
|
||||
assert end_of_emoji != len(text[:2])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"args",
|
||||
[
|
||||
{"action": "click", "value": 1},
|
||||
{"action": "click", "text": "extra"},
|
||||
{"action": "click", "selection_start": 0, "selection_end": 1},
|
||||
{"action": "set_text"},
|
||||
{"action": "set_text", "text": "x" * 65537},
|
||||
{"action": "replace_selection"},
|
||||
{"action": "set_progress"},
|
||||
{"action": "set_progress", "value": True},
|
||||
{"action": "set_progress", "value": "3"},
|
||||
{"action": "set_progress", "value": float("nan")},
|
||||
{"action": "set_progress", "value": float("inf")},
|
||||
{"action": "set_selection", "selection_start": 0},
|
||||
{"action": "set_selection", "selection_start": -1, "selection_end": 1},
|
||||
{"action": "set_selection", "selection_start": 2, "selection_end": 1},
|
||||
{"action": "set_selection", "selection_start": False, "selection_end": 1},
|
||||
{"action": "set_selection", "selection_start": 0, "selection_end": 1.0},
|
||||
{"action": "set_selection", "selection_start": 0, "selection_end": "1"},
|
||||
{"action": "click", "expected_token": "a" * 63},
|
||||
{"action": "click", "expected_token": "G" * 64},
|
||||
{"action": "toggle"},
|
||||
],
|
||||
)
|
||||
def test_control_actions_reject_coercion_wrong_arguments_and_invalid_tokens(args):
|
||||
with pytest.raises(ValidationError):
|
||||
ControlActionRequest.model_validate({"selector": {"resource_id": "control"}, **args})
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"state,extra",
|
||||
[
|
||||
("visible", {}),
|
||||
("absent", {}),
|
||||
("enabled", {}),
|
||||
("disabled", {}),
|
||||
("checked", {}),
|
||||
("unchecked", {}),
|
||||
("selected", {}),
|
||||
("unselected", {}),
|
||||
("text", {"expected_text": " exact 😀 \n"}),
|
||||
("value", {"expected_value": 0}),
|
||||
],
|
||||
)
|
||||
def test_nonstable_waits_allow_short_deadlines_without_overriding_stability_default(state, extra):
|
||||
request = WaitUntilRequest.model_validate(
|
||||
{"selector": {"resource_id": "control"}, "state": state, "timeout_ms": 100, **extra}
|
||||
)
|
||||
assert request.timeout_ms == 100 and request.stable_ms == 300
|
||||
if state == "text":
|
||||
assert request.expected_text == extra["expected_text"]
|
||||
|
||||
|
||||
def test_stability_window_must_fit_the_deadline_only_for_stable_waits():
|
||||
args = {"selector": {"resource_id": "control"}, "state": "stable", "timeout_ms": 100}
|
||||
with pytest.raises(ValidationError, match="stable_ms must fit"):
|
||||
WaitUntilRequest.model_validate(args)
|
||||
assert WaitUntilRequest.model_validate({**args, "stable_ms": 100}).stable_ms == 100
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"name,args",
|
||||
[
|
||||
("wait_until", {"state": "text"}),
|
||||
("wait_until", {"state": "value"}),
|
||||
("wait_until", {"state": "visible", "expected_text": "wrong predicate"}),
|
||||
("wait_until", {"state": "visible", "tolerance": 1}),
|
||||
("wait_until", {"state": "value", "expected_value": float("inf")}),
|
||||
("wait_until", {"state": "visible", "timeout_ms": 0}),
|
||||
("wait_until", {"state": "visible", "poll_interval_ms": 49}),
|
||||
("scroll_to", {"max_scrolls": 0}),
|
||||
("scroll_to", {"max_scrolls": 21}),
|
||||
("scroll_to", {"timeout_ms": 100, "settle_ms": 200}),
|
||||
("scroll_to", {"direction": "diagonal"}),
|
||||
],
|
||||
)
|
||||
def test_condition_and_scroll_requests_reject_incompatible_or_unbounded_arguments(name, args):
|
||||
with pytest.raises(ValidationError):
|
||||
NATIVE_REQUESTS[name].model_validate({**VALID_REQUESTS[name], **args})
|
||||
@@ -0,0 +1,775 @@
|
||||
"""Native semantics with deterministic clocks and no devices or paid models."""
|
||||
|
||||
import asyncio
|
||||
from copy import deepcopy
|
||||
from unittest.mock import AsyncMock
|
||||
from uuid import UUID
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.mcp import native_executor
|
||||
from artemis.mcp.native_executor import execute_native
|
||||
|
||||
SELECTOR = {"resource_id": "control"}
|
||||
CONTAINER = {"resource_id": "list"}
|
||||
TARGET = {"text": "destination"}
|
||||
TOKEN = "a" * 64
|
||||
|
||||
|
||||
def node(**overrides):
|
||||
return {
|
||||
"token": TOKEN,
|
||||
"content_signature": "contents-a",
|
||||
"visible": True,
|
||||
"enabled": True,
|
||||
"checkable": True,
|
||||
"checked": False,
|
||||
"selected": False,
|
||||
"password": False,
|
||||
"text": "exact text",
|
||||
"actions": ["click", "set_progress", "set_text", "scroll_forward"],
|
||||
"range": {"min": 0, "max": 10, "current": 3, "type": 1},
|
||||
**overrides,
|
||||
}
|
||||
|
||||
|
||||
def observed(**overrides):
|
||||
return {"success": True, "execution": "observed", "code": "OK", "node": node(**overrides)}
|
||||
|
||||
|
||||
def missing(code="NOT_FOUND"):
|
||||
return {"success": False, "execution": "not_started", "code": code, "error": code}
|
||||
|
||||
|
||||
def completed(**extra):
|
||||
return {"success": True, "execution": "completed", "code": "OK", **extra}
|
||||
|
||||
|
||||
class Clock:
|
||||
def __init__(self):
|
||||
self.now = 0.0
|
||||
self.sleeps = []
|
||||
|
||||
def monotonic(self):
|
||||
return self.now
|
||||
|
||||
async def sleep(self, seconds):
|
||||
assert seconds > 0
|
||||
self.sleeps.append(seconds)
|
||||
self.now += seconds
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def clock(monkeypatch):
|
||||
fake = Clock()
|
||||
monkeypatch.setattr(native_executor, "monotonic", fake.monotonic)
|
||||
monkeypatch.setattr(native_executor, "sleep", fake.sleep)
|
||||
return fake
|
||||
|
||||
|
||||
async def run(name, args, native):
|
||||
return await execute_native(name, args, 400, 800, native)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("name", ["pinch", "touch_gesture"])
|
||||
async def test_all_gesture_geometry_is_validated_before_a_single_native_dispatch(name):
|
||||
args = (
|
||||
{
|
||||
"center": [200, 400],
|
||||
"start_span": 40,
|
||||
"end_span": 80,
|
||||
"rotation_degrees": 45,
|
||||
"duration_ms": 900,
|
||||
"coordinate_space": "pixel",
|
||||
"target_description": "observed control",
|
||||
}
|
||||
if name == "pinch"
|
||||
else {
|
||||
"frames": [
|
||||
{"at_ms": 0, "points": [[100, 100]]},
|
||||
{"at_ms": 300, "points": [[120, 160]]},
|
||||
{"at_ms": 900, "points": [[140, 220]]},
|
||||
],
|
||||
"coordinate_space": "pixel",
|
||||
"target_description": "observed control",
|
||||
}
|
||||
)
|
||||
native = AsyncMock(return_value=completed(native_marker="original receipt"))
|
||||
result = await run(name, args, native)
|
||||
|
||||
assert result["success"] and result["execution"] == "completed"
|
||||
assert result["verification"] == "pending_observation"
|
||||
assert result["native_receipt"]["native_marker"] == "original receipt"
|
||||
native.assert_awaited_once()
|
||||
command, sent = native.await_args.args
|
||||
assert command == "touch_gesture"
|
||||
assert UUID(sent["operation_id"])
|
||||
assert sent["frames"] == result["plan"]["frames"]
|
||||
assert sent["screen_size"] == [400, 800]
|
||||
assert native.await_args.kwargs == {"timeout": 5.9}
|
||||
|
||||
invalid = deepcopy(args)
|
||||
if name == "pinch":
|
||||
invalid["center"] = [0, 0]
|
||||
else:
|
||||
invalid["frames"][-1]["points"] = [[401, 220]]
|
||||
never = AsyncMock()
|
||||
refused = await run(name, invalid, never)
|
||||
assert not refused["success"] and refused["execution"] == "not_started"
|
||||
never.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_touch_timeout_never_replays_the_input():
|
||||
native = AsyncMock(side_effect=TimeoutError("reply lost"))
|
||||
result = await run(
|
||||
"touch_gesture",
|
||||
{
|
||||
"frames": [{"at_ms": 0, "points": [[10, 10]]}, {"at_ms": 20, "points": [[20, 30]]}],
|
||||
"target_description": "observed control",
|
||||
"require_visible": "required control",
|
||||
},
|
||||
native,
|
||||
)
|
||||
assert not result["success"] and result["execution"] == "unknown"
|
||||
assert result["code"] == "TIMEOUT"
|
||||
native.assert_awaited_once()
|
||||
assert native.await_args.args[1]["require_visible"] == "required control"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_control_keeps_native_evidence_and_omits_null_selector_fields():
|
||||
receipt = observed()
|
||||
native = AsyncMock(return_value=receipt)
|
||||
result = await run("read_control", {"selector": {**SELECTOR, "text": None}}, native)
|
||||
|
||||
native.assert_awaited_once_with("inspect_control", {"selector": SELECTOR}, timeout=None)
|
||||
assert result["node"] == receipt["node"]
|
||||
assert result["native_receipt"] == receipt
|
||||
assert result["execution"] == "observed" and result["action"] == "read_control"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_control_action_uses_fresh_token_preserves_whitespace_and_does_not_verify_business_value():
|
||||
before = observed()
|
||||
after = node(text="different accepted text")
|
||||
receipt = completed(node_before=before["node"], node_after=after)
|
||||
native = AsyncMock(side_effect=[before, receipt])
|
||||
result = await run(
|
||||
"control_action", {"selector": SELECTOR, "action": "set_text", "text": " exact\n"}, native
|
||||
)
|
||||
|
||||
assert native.await_args_list[1].args == (
|
||||
"control_action",
|
||||
{"selector": SELECTOR, "action": "set_text", "text": " exact\n", "expected_token": TOKEN},
|
||||
)
|
||||
assert native.await_count == 2
|
||||
assert result["success"] and result["verification"] == "pending_observation"
|
||||
assert result["node_before"] == before["node"]
|
||||
assert result["node_after"] == after
|
||||
assert result["native_receipt"] == receipt
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"snapshot,request_extra,code",
|
||||
[
|
||||
(node(), {"expected_token": "b" * 64}, "STALE"),
|
||||
(node(actions=["long_click"]), {}, "UNSUPPORTED"),
|
||||
(node(enabled=False), {}, "DISABLED"),
|
||||
(node(token=None), {}, "INVALID_RESPONSE"),
|
||||
],
|
||||
)
|
||||
async def test_control_action_refuses_stale_unsupported_or_disabled_without_input(
|
||||
snapshot, request_extra, code
|
||||
):
|
||||
native = AsyncMock(return_value={**observed(), "node": snapshot})
|
||||
result = await run(
|
||||
"control_action", {"selector": SELECTOR, "action": "click", **request_extra}, native
|
||||
)
|
||||
assert not result["success"] and result["code"] == code
|
||||
assert result["execution"] == "not_started"
|
||||
assert result["node_before"] == snapshot
|
||||
native.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"native_range,value",
|
||||
[
|
||||
({"min": 0, "max": 10, "current": 3}, 11),
|
||||
({"min": 0, "max": 10, "current": float("nan")}, 3),
|
||||
({"min": 0, "max": 10, "current": 3, "type": 0}, 3.5),
|
||||
],
|
||||
)
|
||||
async def test_set_progress_refuses_invalid_live_range_before_input(native_range, value):
|
||||
native = AsyncMock(return_value=observed(range=native_range))
|
||||
result = await run(
|
||||
"control_action", {"selector": SELECTOR, "action": "set_progress", "value": value}, native
|
||||
)
|
||||
assert not result["success"] and result["execution"] == "not_started"
|
||||
native.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_control_transport_failure_is_unknown_and_never_replayed():
|
||||
native = AsyncMock(side_effect=[observed(), OSError("lost reply")])
|
||||
result = await run("control_action", {"selector": SELECTOR, "action": "click"}, native)
|
||||
assert result["code"] == "NATIVE_ERROR" and result["execution"] == "unknown"
|
||||
assert not result["success"] and native.await_count == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"receipt",
|
||||
[
|
||||
"legacy unstructured success",
|
||||
None,
|
||||
{"success": True, "execution": [], "code": "OK"},
|
||||
{"success": True, "execution": "unknown", "code": "OK"},
|
||||
{"success": True, "execution": "completed"},
|
||||
],
|
||||
)
|
||||
async def test_malformed_write_receipts_are_unknown_and_never_replayed(receipt):
|
||||
native = AsyncMock(side_effect=[observed(), receipt])
|
||||
result = await run("control_action", {"selector": SELECTOR, "action": "click"}, native)
|
||||
assert not result["success"] and result["execution"] == "unknown"
|
||||
assert result["code"] == "INVALID_RESPONSE" and native.await_count == 2
|
||||
assert result["native_receipt"] == receipt
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wait_observes_delayed_condition_without_operating_or_sleeping_after_success(clock):
|
||||
native = AsyncMock(side_effect=[missing(), observed(visible=False), observed()])
|
||||
result = await run(
|
||||
"wait_until",
|
||||
{"selector": SELECTOR, "state": "visible", "timeout_ms": 1000, "poll_interval_ms": 100},
|
||||
native,
|
||||
)
|
||||
assert result["success"] and result["execution"] == "observed"
|
||||
assert result["attempts"] == 3 and clock.now == pytest.approx(0.2)
|
||||
assert [call.args[0] for call in native.await_args_list] == ["inspect_control"] * 3
|
||||
assert [call.kwargs["timeout"] for call in native.await_args_list] == pytest.approx(
|
||||
[1, 0.9, 0.8]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("changed_field", ["token", "content_signature"])
|
||||
async def test_stability_restarts_on_identity_or_descendant_redraw(clock, changed_field):
|
||||
change = {changed_field: "changed"}
|
||||
native = AsyncMock(
|
||||
side_effect=[observed(), observed(**change), observed(**change), observed(**change)]
|
||||
)
|
||||
result = await run(
|
||||
"wait_until",
|
||||
{
|
||||
"selector": SELECTOR,
|
||||
"state": "stable",
|
||||
"stable_ms": 190,
|
||||
"timeout_ms": 1000,
|
||||
"poll_interval_ms": 100,
|
||||
},
|
||||
native,
|
||||
)
|
||||
assert result["success"] and result["attempts"] == 4
|
||||
assert clock.now == pytest.approx(0.3)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_zero_stability_window_still_requires_two_actual_reads(clock):
|
||||
native = AsyncMock(return_value=observed())
|
||||
result = await run(
|
||||
"wait_until", {"selector": SELECTOR, "state": "stable", "stable_ms": 0}, native
|
||||
)
|
||||
assert result["success"] and result["attempts"] == 2
|
||||
assert len(clock.sleeps) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("receipt", [missing(), observed(visible=False)])
|
||||
async def test_absent_accepts_only_not_found_or_unique_explicitly_invisible(clock, receipt):
|
||||
native = AsyncMock(return_value=receipt)
|
||||
result = await run("wait_until", {"selector": SELECTOR, "state": "absent"}, native)
|
||||
assert result["success"] and result["execution"] == "observed"
|
||||
native.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"error", ["AMBIGUOUS", "UNSUPPORTED", "LIMIT_EXCEEDED", "INVALID_RESPONSE"]
|
||||
)
|
||||
async def test_absent_rejects_nontransient_inspection_errors_immediately(clock, error):
|
||||
native = AsyncMock(return_value=missing(error))
|
||||
result = await run("wait_until", {"selector": SELECTOR, "state": "absent"}, native)
|
||||
assert not result["success"] and result["code"] == error
|
||||
native.assert_awaited_once()
|
||||
assert not clock.sleeps
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("error", ["UNAVAILABLE", "READ_FAILED"])
|
||||
async def test_unreadable_windows_time_out_without_proving_absence(clock, error):
|
||||
native = AsyncMock(return_value=missing(error))
|
||||
result = await run(
|
||||
"wait_until",
|
||||
{"selector": SELECTOR, "state": "absent", "timeout_ms": 300, "poll_interval_ms": 100},
|
||||
native,
|
||||
)
|
||||
assert not result["success"] and result["code"] == "TIMEOUT"
|
||||
assert result["native_receipt"]["code"] == error
|
||||
assert result["attempts"] == 3 and clock.now == pytest.approx(0.3)
|
||||
assert [sample["code"] for sample in result["observations"]] == [error] * 3
|
||||
assert [call.args[0] for call in native.await_args_list] == ["inspect_control"] * 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("error", ["UNAVAILABLE", "READ_FAILED"])
|
||||
@pytest.mark.parametrize("state,receipt", [("visible", observed()), ("absent", missing())])
|
||||
async def test_wait_recovers_only_after_a_reliable_observation(clock, error, state, receipt):
|
||||
native = AsyncMock(side_effect=[missing(error), receipt])
|
||||
result = await run(
|
||||
"wait_until",
|
||||
{"selector": SELECTOR, "state": state, "timeout_ms": 500, "poll_interval_ms": 100},
|
||||
native,
|
||||
)
|
||||
assert result["success"] and result["execution"] == "observed"
|
||||
assert result["attempts"] == 2 and clock.now == pytest.approx(0.1)
|
||||
assert result["observations"][0]["code"] == error
|
||||
assert result["native_receipt"] == receipt
|
||||
assert [call.args[0] for call in native.await_args_list] == ["inspect_control"] * 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("error", ["UNAVAILABLE", "READ_FAILED"])
|
||||
async def test_transient_read_failure_restarts_the_full_stability_window(clock, error):
|
||||
native = AsyncMock(side_effect=[observed(), missing(error), observed(), observed(), observed()])
|
||||
result = await run(
|
||||
"wait_until",
|
||||
{
|
||||
"selector": SELECTOR,
|
||||
"state": "stable",
|
||||
"stable_ms": 190,
|
||||
"timeout_ms": 1000,
|
||||
"poll_interval_ms": 100,
|
||||
},
|
||||
native,
|
||||
)
|
||||
assert result["success"] and result["attempts"] == 5
|
||||
assert clock.now == pytest.approx(0.4)
|
||||
assert result["observations"][1]["code"] == error
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_absent_transport_failure_is_not_absence(clock):
|
||||
native = AsyncMock(side_effect=OSError("unavailable"))
|
||||
result = await run("wait_until", {"selector": SELECTOR, "state": "absent"}, native)
|
||||
assert not result["success"] and result["code"] == "NATIVE_ERROR"
|
||||
native.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("receipt", [{**observed(), "candidate_count": 2}, observed(visible=None)])
|
||||
async def test_absent_rejects_nonunique_or_unknown_visibility_observation(clock, receipt):
|
||||
native = AsyncMock(return_value=receipt)
|
||||
result = await run("wait_until", {"selector": SELECTOR, "state": "absent"}, native)
|
||||
assert not result["success"]
|
||||
assert result["code"] in {"AMBIGUOUS", "INVALID_STATE"}
|
||||
native.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"state,args,snapshot,success",
|
||||
[
|
||||
("text", {"expected_text": " exact text "}, node(), False),
|
||||
("text", {"expected_text": "exact text"}, node(), True),
|
||||
("value", {"expected_value": 3.1, "tolerance": 0.2}, node(), True),
|
||||
(
|
||||
"value",
|
||||
{"expected_value": 3},
|
||||
node(range={"min": 0, "max": float("inf"), "current": 3}),
|
||||
False,
|
||||
),
|
||||
("unchecked", {}, node(checkable=False, checked=False), False),
|
||||
("checked", {}, node(checked=True), True),
|
||||
("disabled", {}, node(enabled=False), True),
|
||||
("unselected", {}, node(selected=False), True),
|
||||
],
|
||||
)
|
||||
async def test_wait_predicates_use_meaningful_exact_native_values(
|
||||
clock, state, args, snapshot, success
|
||||
):
|
||||
native = AsyncMock(return_value={**observed(), "node": snapshot})
|
||||
result = await run(
|
||||
"wait_until",
|
||||
{"selector": SELECTOR, "state": state, "timeout_ms": 400, "stable_ms": 0, **args},
|
||||
native,
|
||||
)
|
||||
assert result["success"] is success
|
||||
assert all(call.args[0] == "inspect_control" for call in native.await_args_list)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wait_deadline_is_bounded_and_preserves_observations(clock):
|
||||
native = AsyncMock(return_value=observed(visible=False))
|
||||
result = await run(
|
||||
"wait_until",
|
||||
{"selector": SELECTOR, "state": "visible", "timeout_ms": 350, "poll_interval_ms": 100},
|
||||
native,
|
||||
)
|
||||
assert not result["success"] and result["code"] == "TIMEOUT"
|
||||
assert clock.now == pytest.approx(0.35) and native.await_count == 4
|
||||
assert len(result["observations"]) == 4
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wait_rejects_condition_returned_after_monotonic_deadline(clock):
|
||||
async def native(command, args, *, timeout=None):
|
||||
clock.now += 0.6
|
||||
return observed()
|
||||
|
||||
result = await run(
|
||||
"wait_until", {"selector": SELECTOR, "state": "visible", "timeout_ms": 500}, native
|
||||
)
|
||||
assert not result["success"] and result["code"] == "TIMEOUT"
|
||||
assert result["attempts"] == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wait_stops_a_nonreturning_native_read_at_its_deadline(clock):
|
||||
calls = []
|
||||
|
||||
async def native(command, args, *, timeout=None):
|
||||
calls.append(command)
|
||||
await asyncio.Event().wait()
|
||||
|
||||
result = await run(
|
||||
"wait_until",
|
||||
{"selector": SELECTOR, "state": "visible", "timeout_ms": 20, "stable_ms": 0},
|
||||
native,
|
||||
)
|
||||
assert not result["success"] and result["code"] == "TIMEOUT"
|
||||
assert calls == ["inspect_control"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_long_wait_bounds_evidence_memory_while_retaining_first_and_last_reads(clock):
|
||||
native = AsyncMock(return_value=observed(visible=False))
|
||||
result = await run(
|
||||
"wait_until",
|
||||
{"selector": SELECTOR, "state": "visible", "timeout_ms": 4000, "poll_interval_ms": 50},
|
||||
native,
|
||||
)
|
||||
assert result["attempts"] > 64 and len(result["observations"]) == 64
|
||||
assert result["observations"][0]["at_ms"] == 0
|
||||
assert result["observations"][-1]["at_ms"] > 3000
|
||||
|
||||
|
||||
class ScrollNative:
|
||||
def __init__(self, clock, *, delayed_target=None, progress=True, receipt=None):
|
||||
self.clock = clock
|
||||
self.delayed_target = delayed_target
|
||||
self.progress = progress
|
||||
self.receipt = receipt or completed()
|
||||
self.calls = []
|
||||
self.writes = 0
|
||||
|
||||
async def __call__(self, command, args, *, timeout=None):
|
||||
self.calls.append((command, deepcopy(args), timeout, self.clock.now))
|
||||
if command == "control_action":
|
||||
self.writes += 1
|
||||
return self.receipt
|
||||
if args["selector"] == TARGET:
|
||||
# An unscoped query would spuriously find a same-text external node.
|
||||
if args.get("scope") != CONTAINER:
|
||||
return observed(text="external false positive")
|
||||
if (
|
||||
self.writes
|
||||
and self.delayed_target is not None
|
||||
and self.clock.now >= self.delayed_target
|
||||
):
|
||||
return observed(text="destination")
|
||||
return missing()
|
||||
assert args == {"selector": CONTAINER}
|
||||
changed = self.writes and self.progress and self.clock.now >= 0.1
|
||||
return observed(
|
||||
token=("b" if self.writes else "a") * 64,
|
||||
content_signature=f"page-{self.writes}" if changed else "contents-a",
|
||||
)
|
||||
|
||||
|
||||
def scroll_request(**extra):
|
||||
return {"container": CONTAINER, "target": TARGET, "settle_ms": 300, "timeout_ms": 2000, **extra}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scroll_waits_for_delayed_scoped_target_after_content_changes(clock):
|
||||
native = ScrollNative(clock, delayed_target=0.2)
|
||||
result = await run("scroll_to", scroll_request(), native)
|
||||
assert result["success"] and result["execution"] == "completed"
|
||||
assert native.writes == 1 and clock.now >= 0.4 - 1e-9
|
||||
assert result["node"]["text"] == "destination"
|
||||
assert result["scroll_results"][0]["native_receipt"]["success"]
|
||||
queries = [
|
||||
args
|
||||
for command, args, _, _ in native.calls
|
||||
if command == "inspect_control" and args["selector"] == TARGET
|
||||
]
|
||||
assert queries and all(query["scope"] == CONTAINER for query in queries)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scroll_token_change_without_content_change_is_no_progress(clock):
|
||||
native = ScrollNative(clock, progress=False)
|
||||
result = await run("scroll_to", scroll_request(), native)
|
||||
assert not result["success"] and result["code"] == "NO_PROGRESS"
|
||||
assert result["execution"] == "completed" and native.writes == 1
|
||||
assert clock.now == pytest.approx(0.5)
|
||||
assert result["scroll_results"][0]["native_receipt"]["success"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scroll_preserves_each_receipt_and_refreshes_token_with_hard_input_limit(clock):
|
||||
native = ScrollNative(clock)
|
||||
result = await run("scroll_to", scroll_request(max_scrolls=2), native)
|
||||
assert result["code"] == "MAX_SCROLLS" and not result["success"]
|
||||
assert native.writes == 2 and len(result["scroll_results"]) == 2
|
||||
writes = [(args, at) for command, args, _, at in native.calls if command == "control_action"]
|
||||
assert writes[0][0]["expected_token"] == TOKEN
|
||||
assert writes[1][0]["expected_token"] == "b" * 64
|
||||
assert writes[1][1] >= 0.3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("execution", ["unknown", "cancelled"])
|
||||
async def test_scroll_unknown_or_cancelled_input_is_never_repeated(clock, execution):
|
||||
receipt = {"success": False, "execution": execution, "code": "TIMEOUT", "native_marker": "kept"}
|
||||
native = ScrollNative(clock, receipt=receipt)
|
||||
result = await run("scroll_to", scroll_request(), native)
|
||||
assert not result["success"] and result["execution"] == execution
|
||||
assert result["scroll_results"][0]["native_receipt"] == receipt
|
||||
assert native.writes == 1 and not clock.sleeps
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scroll_boundary_rejection_is_no_progress_without_retry(clock):
|
||||
native = ScrollNative(clock, receipt=missing("ACTION_REJECTED"))
|
||||
result = await run("scroll_to", scroll_request(), native)
|
||||
assert result["code"] == "NO_PROGRESS" and native.writes == 1
|
||||
assert result["scroll_results"][0]["native_receipt"]["code"] == "ACTION_REJECTED"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("which", ["container", "target"])
|
||||
async def test_scroll_refuses_ambiguous_container_or_scoped_target_before_input(clock, which):
|
||||
calls = []
|
||||
|
||||
async def native(command, args, *, timeout=None):
|
||||
calls.append(command)
|
||||
assert command == "inspect_control"
|
||||
if args["selector"] == (CONTAINER if which == "container" else TARGET):
|
||||
return missing("AMBIGUOUS")
|
||||
return observed()
|
||||
|
||||
result = await run("scroll_to", scroll_request(), native)
|
||||
assert not result["success"] and result["code"] == "AMBIGUOUS"
|
||||
assert "control_action" not in calls
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"write_receipt,execution",
|
||||
[
|
||||
(completed(native_marker="late accepted input"), "completed"),
|
||||
(
|
||||
{
|
||||
"success": False,
|
||||
"execution": "unknown",
|
||||
"code": "TIMEOUT",
|
||||
"native_marker": "late uncertain receipt",
|
||||
},
|
||||
"unknown",
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_scroll_drains_dispatched_input_past_search_deadline_without_more_work(
|
||||
clock, write_receipt, execution
|
||||
):
|
||||
calls = []
|
||||
events = []
|
||||
pending_input = None
|
||||
|
||||
async def deliver_input():
|
||||
# Device input survives a lost HTTP waiter; cancelling its await cannot undo it.
|
||||
await asyncio.sleep(0.12)
|
||||
clock.now = 0.12
|
||||
events.append("input_delivered")
|
||||
return write_receipt
|
||||
|
||||
async def native(command, args, *, timeout=None):
|
||||
nonlocal pending_input
|
||||
calls.append((command, deepcopy(args), timeout))
|
||||
if command == "inspect_control":
|
||||
return missing() if args["selector"] == TARGET else observed()
|
||||
pending_input = asyncio.create_task(deliver_input())
|
||||
return await asyncio.wait_for(asyncio.shield(pending_input), timeout=timeout)
|
||||
|
||||
try:
|
||||
result = await run(
|
||||
"scroll_to", scroll_request(timeout_ms=50, settle_ms=50, max_scrolls=3), native
|
||||
)
|
||||
events.append("tool_returned")
|
||||
finally:
|
||||
if pending_input is not None:
|
||||
await pending_input
|
||||
|
||||
assert events == ["input_delivered", "tool_returned"]
|
||||
assert not result["success"] and result["code"] == "TIMEOUT"
|
||||
assert result["execution"] == execution
|
||||
assert result["scroll_results"][0]["native_receipt"] == write_receipt
|
||||
assert [command for command, _, _ in calls] == [
|
||||
"inspect_control",
|
||||
"inspect_control",
|
||||
"control_action",
|
||||
]
|
||||
assert calls[-1][2] == 5.0
|
||||
assert not clock.sleeps
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scroll_timeout_caps_total_search_even_when_content_changes(clock):
|
||||
native = ScrollNative(clock)
|
||||
result = await run("scroll_to", scroll_request(timeout_ms=350, settle_ms=300), native)
|
||||
assert result["code"] == "TIMEOUT" and not result["success"]
|
||||
assert clock.now == pytest.approx(0.35) and native.writes <= 2
|
||||
assert len(result["scroll_results"]) == native.writes
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scroll_requires_visible_target_not_merely_an_existing_scoped_node(clock):
|
||||
backend = ScrollNative(clock, delayed_target=0.2)
|
||||
|
||||
async def native(command, args, *, timeout=None):
|
||||
result = await backend(command, args, timeout=timeout)
|
||||
return (
|
||||
observed(text="destination", visible=False)
|
||||
if result.get("code") == "NOT_FOUND"
|
||||
else result
|
||||
)
|
||||
|
||||
result = await run("scroll_to", scroll_request(), native)
|
||||
assert result["success"] and backend.writes == 1
|
||||
assert clock.now >= 0.4 - 1e-9
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scroll_does_not_send_next_input_while_content_is_animating(clock):
|
||||
writes = []
|
||||
|
||||
async def native(command, args, *, timeout=None):
|
||||
if command == "control_action":
|
||||
writes.append(clock.now)
|
||||
return completed()
|
||||
if args["selector"] == TARGET:
|
||||
return missing()
|
||||
page = min(int(clock.now * 10 + 1e-6), 6) if writes else 0
|
||||
return observed(content_signature=f"animation-{page}")
|
||||
|
||||
result = await run("scroll_to", scroll_request(max_scrolls=2), native)
|
||||
assert result["code"] == "NO_PROGRESS" and len(writes) == 2
|
||||
assert writes[1] >= 0.9 - 1e-9
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scroll_transient_visible_target_during_animation_is_not_success(clock):
|
||||
writes = []
|
||||
|
||||
async def native(command, args, *, timeout=None):
|
||||
if command == "control_action":
|
||||
writes.append(clock.now)
|
||||
return completed()
|
||||
if args["selector"] == TARGET:
|
||||
return observed() if writes and 0.1 <= clock.now < 0.2 else missing()
|
||||
return observed(content_signature="after" if writes and clock.now >= 0.25 else "before")
|
||||
|
||||
result = await run("scroll_to", scroll_request(max_scrolls=1), native)
|
||||
assert result["code"] == "MAX_SCROLLS" and not result["success"]
|
||||
assert len(writes) == 1 and clock.now >= 0.55 - 1e-9
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scroll_tracks_target_outside_bounded_container_signature_sample(clock):
|
||||
writes = []
|
||||
|
||||
async def native(command, args, *, timeout=None):
|
||||
if command == "control_action":
|
||||
writes.append(clock.now)
|
||||
return completed()
|
||||
if args["selector"] == TARGET:
|
||||
return observed() if writes and 0.4 - 1e-9 <= clock.now < 0.45 - 1e-9 else missing()
|
||||
return observed(content_signature="after" if writes and clock.now >= 0.1 else "before")
|
||||
|
||||
result = await run("scroll_to", scroll_request(max_scrolls=1), native)
|
||||
assert result["code"] == "MAX_SCROLLS" and not result["success"]
|
||||
assert len(writes) == 1 and clock.now >= 0.75 - 1e-9
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scroll_read_failure_after_accepted_input_preserves_receipt_without_more_input(clock):
|
||||
backend = ScrollNative(clock)
|
||||
|
||||
async def native(command, args, *, timeout=None):
|
||||
if command == "inspect_control" and backend.writes:
|
||||
raise OSError("read unavailable after accepted input")
|
||||
return await backend(command, args, timeout=timeout)
|
||||
|
||||
result = await run("scroll_to", scroll_request(), native)
|
||||
assert not result["success"] and result["execution"] == "completed"
|
||||
assert result["code"] == "NATIVE_ERROR" and backend.writes == 1
|
||||
assert result["scroll_results"][0]["native_receipt"]["success"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancelled_scroll_settling_never_sends_another_input(clock, monkeypatch):
|
||||
backend = ScrollNative(clock)
|
||||
|
||||
async def cancelled_sleep(seconds):
|
||||
raise asyncio.CancelledError
|
||||
|
||||
monkeypatch.setattr(native_executor, "sleep", cancelled_sleep)
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await run("scroll_to", scroll_request(), backend)
|
||||
assert backend.writes == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("name", ["control_action", "wait_until", "scroll_to", "touch_gesture"])
|
||||
async def test_cancellation_propagates_without_replaying_native_work(clock, name):
|
||||
requests = {
|
||||
"control_action": {"selector": SELECTOR, "action": "click"},
|
||||
"wait_until": {"selector": SELECTOR, "state": "visible"},
|
||||
"scroll_to": scroll_request(),
|
||||
"touch_gesture": {
|
||||
"frames": [{"at_ms": 0, "points": [[1, 1]]}, {"at_ms": 20, "points": [[2, 3]]}],
|
||||
"target_description": "observed control",
|
||||
},
|
||||
}
|
||||
inputs = []
|
||||
|
||||
async def native(command, args, *, timeout=None):
|
||||
if command == "inspect_control":
|
||||
if name == "wait_until":
|
||||
raise asyncio.CancelledError
|
||||
return missing() if args["selector"] == TARGET else observed()
|
||||
inputs.append(command)
|
||||
raise asyncio.CancelledError
|
||||
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await run(name, requests[name], native)
|
||||
assert inputs == (
|
||||
[]
|
||||
if name == "wait_until"
|
||||
else ["touch_gesture" if name == "touch_gesture" else "control_action"]
|
||||
)
|
||||
@@ -0,0 +1,270 @@
|
||||
"""Mixed native sequences exercise production preflight and dispatch over mocks."""
|
||||
|
||||
from copy import deepcopy
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.mcp.action_manifest import DEVICE_ACTIONS
|
||||
from artemis.mcp.actuators import MockActuator
|
||||
|
||||
|
||||
SELECTOR = {"resource_id": "fixture:id/editor"}
|
||||
TOKEN = "a" * 64
|
||||
|
||||
|
||||
def wait_step():
|
||||
return {"action": "wait_until", "selector": SELECTOR, "state": "visible", "timeout_ms": 100}
|
||||
|
||||
|
||||
def control_step():
|
||||
return {
|
||||
"action": "control_action",
|
||||
"request": {"selector": SELECTOR, "action": "set_text", "text": " exact 😀 \n"},
|
||||
}
|
||||
|
||||
|
||||
def touch_step():
|
||||
return {
|
||||
"action": "touch_gesture",
|
||||
"frames": [
|
||||
{"at_ms": 0, "points": [[250, 500]]},
|
||||
{"at_ms": 150, "points": [[250, 500]]},
|
||||
{"at_ms": 400, "points": [[500, 750]]},
|
||||
],
|
||||
"coordinate_space": "normalized",
|
||||
"target_description": "observed canvas",
|
||||
}
|
||||
|
||||
|
||||
def click_step():
|
||||
return {"action": "click", "target": [50, 100], "target_description": "observed button"}
|
||||
|
||||
|
||||
def observed():
|
||||
return {
|
||||
"success": True,
|
||||
"execution": "observed",
|
||||
"code": "OK",
|
||||
"node": {
|
||||
"token": TOKEN,
|
||||
"content_signature": "b" * 64,
|
||||
"visible": True,
|
||||
"enabled": True,
|
||||
"editable": True,
|
||||
"password": False,
|
||||
"text": " A😀B ",
|
||||
"selection_start": 1,
|
||||
"selection_end": 3,
|
||||
"actions": ["set_text", "set_selection"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def sequence_limit(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"artemis.config.agent.load_agent_config",
|
||||
lambda: SimpleNamespace(
|
||||
pro=SimpleNamespace(execution=SimpleNamespace(max_burst_actions=10))
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def actuator_with_native(capabilities=DEVICE_ACTIONS):
|
||||
actuator = MockActuator(width=720, height=1600, capabilities=capabilities)
|
||||
|
||||
async def native(command, args, *, timeout=None):
|
||||
if command == "inspect_control":
|
||||
return observed()
|
||||
return {
|
||||
"success": True,
|
||||
"execution": "completed",
|
||||
"code": "OK",
|
||||
"native_marker": command,
|
||||
}
|
||||
|
||||
actuator.controller.native_request = AsyncMock(side_effect=native)
|
||||
actuator.controller.tap_at = AsyncMock(return_value=SimpleNamespace(error=None))
|
||||
actuator.controller.swipe_coords = AsyncMock()
|
||||
return actuator
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"bad_member",
|
||||
[
|
||||
{**wait_step(), "timeout_ms": 0},
|
||||
{**wait_step(), "state": "text"},
|
||||
{**wait_step(), "timeout_ms": 3001},
|
||||
{"action": "control_action", "request": {"selector": SELECTOR, "action": "set_progress"}},
|
||||
{
|
||||
"action": "control_action",
|
||||
"request": {"selector": SELECTOR, "action": "set_text", "text": "x" * 1025},
|
||||
},
|
||||
{
|
||||
"action": "control_action",
|
||||
"request": {"selector": SELECTOR, "action": "click", "text": "unexpected"},
|
||||
},
|
||||
{
|
||||
**touch_step(),
|
||||
"frames": [
|
||||
{"at_ms": 0, "points": [[100, 100]]},
|
||||
{"at_ms": 400, "points": [[1001, 100]]},
|
||||
],
|
||||
},
|
||||
{
|
||||
**touch_step(),
|
||||
"frames": [
|
||||
{"at_ms": 0, "points": [[100, 100], [300, 100]]},
|
||||
{"at_ms": 400, "points": [[300, 100], [100, 100]]},
|
||||
],
|
||||
},
|
||||
{
|
||||
**touch_step(),
|
||||
"frames": [
|
||||
{"at_ms": 0, "points": [[100, 100]]},
|
||||
{"at_ms": 400, "points": [[200, 200], [300, 300]]},
|
||||
],
|
||||
},
|
||||
],
|
||||
)
|
||||
async def test_all_native_parameters_and_paths_are_preflighted_before_even_the_first_click(
|
||||
bad_member,
|
||||
):
|
||||
actuator = actuator_with_native()
|
||||
result = await actuator.action_sequence(
|
||||
[click_step(), wait_step(), control_step(), touch_step(), bad_member]
|
||||
)
|
||||
assert not result.ok and result.code.value == "INVALID_ARGS"
|
||||
actuator.controller.native_request.assert_not_awaited()
|
||||
actuator.controller.tap_at.assert_not_awaited()
|
||||
actuator.controller.swipe_coords.assert_not_awaited()
|
||||
assert actuator.action_history == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("missing_capability", ["wait_until", "control_action", "touch_gesture"])
|
||||
async def test_every_native_member_capability_is_checked_before_any_input(missing_capability):
|
||||
actuator = actuator_with_native(DEVICE_ACTIONS - {missing_capability})
|
||||
result = await actuator.action_sequence(
|
||||
[click_step(), wait_step(), control_step(), touch_step()]
|
||||
)
|
||||
assert not result.ok and result.code.value == "UNSUPPORTED"
|
||||
assert result.metadata == {"execution": "not_started", "side_effect": "none"}
|
||||
actuator.controller.native_request.assert_not_awaited()
|
||||
actuator.controller.tap_at.assert_not_awaited()
|
||||
actuator.controller.swipe_coords.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mixed_wait_control_and_touch_keep_native_receipts_and_exact_text_in_order():
|
||||
actuator = actuator_with_native()
|
||||
steps = [wait_step(), control_step(), touch_step()]
|
||||
original = deepcopy(steps)
|
||||
result = await actuator.action_sequence(steps)
|
||||
assert result.ok and result.metadata["side_effect"] == "possible"
|
||||
assert steps == original
|
||||
calls = actuator.controller.native_request.await_args_list
|
||||
assert [call.args[0] for call in calls] == [
|
||||
"inspect_control",
|
||||
"inspect_control",
|
||||
"control_action",
|
||||
"touch_gesture",
|
||||
]
|
||||
assert calls[2].args[1]["text"] == " exact 😀 \n"
|
||||
assert calls[2].args[1]["expected_token"] == TOKEN
|
||||
receipts = result.metadata["sequence_results"]
|
||||
assert [item["action"] for item in receipts] == [
|
||||
"wait_until",
|
||||
"control_action",
|
||||
"touch_gesture",
|
||||
]
|
||||
assert receipts[0]["read_only"] and receipts[0]["side_effect"] == "none"
|
||||
assert receipts[0]["native_execution"] == "observed"
|
||||
assert receipts[0]["node"]["selection_end"] == 3
|
||||
assert receipts[1]["native_receipt"]["native_marker"] == "control_action"
|
||||
assert receipts[2]["native_receipt"]["native_marker"] == "touch_gesture"
|
||||
assert receipts[0]["ended_ms"] <= receipts[1]["started_ms"] <= receipts[2]["started_ms"]
|
||||
actuator.controller.tap_at.assert_not_awaited()
|
||||
actuator.controller.swipe_coords.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("failure", [False, True])
|
||||
async def test_readonly_native_wait_never_counts_as_an_input_side_effect(failure):
|
||||
actuator = actuator_with_native()
|
||||
if failure:
|
||||
actuator.controller.native_request.return_value = None
|
||||
actuator.controller.native_request.side_effect = lambda *args, **kwargs: {
|
||||
"success": False,
|
||||
"execution": "not_started",
|
||||
"code": "UNAVAILABLE",
|
||||
"error": "No active native tree",
|
||||
}
|
||||
result = await actuator.action_sequence([wait_step()])
|
||||
assert result.ok is not failure
|
||||
assert result.metadata["side_effect"] == "none"
|
||||
receipt = result.metadata["sequence_results"][0]
|
||||
assert receipt["read_only"] and receipt["side_effect"] == "none"
|
||||
assert receipt["native_execution"] == ("not_started" if failure else "observed")
|
||||
assert [call.args[0] for call in actuator.controller.native_request.await_args_list] == [
|
||||
"inspect_control"
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("unknown_member", ["control_action", "touch_gesture"])
|
||||
async def test_unknown_native_input_stops_every_remaining_member_and_preserves_original_receipt(
|
||||
unknown_member,
|
||||
):
|
||||
actuator = actuator_with_native()
|
||||
uncertain = {
|
||||
"success": False,
|
||||
"execution": "unknown",
|
||||
"code": "TRANSPORT_ERROR",
|
||||
"error": "Reply lost; do not replay input",
|
||||
"native_marker": "unacknowledged input",
|
||||
}
|
||||
|
||||
async def native(command, args, *, timeout=None):
|
||||
return observed() if command == "inspect_control" else uncertain
|
||||
|
||||
actuator.controller.native_request.side_effect = native
|
||||
member = control_step() if unknown_member == "control_action" else touch_step()
|
||||
result = await actuator.action_sequence([wait_step(), member, click_step(), touch_step()])
|
||||
assert not result.ok and result.metadata["side_effect"] == "possible"
|
||||
receipts = result.metadata["sequence_results"]
|
||||
assert receipts[1]["execution"] == "unknown"
|
||||
assert receipts[1]["native_execution"] == "unknown"
|
||||
assert receipts[1]["native_receipt"] == uncertain
|
||||
assert [item["execution"] for item in receipts[2:]] == ["not_started", "not_started"]
|
||||
writes = [
|
||||
call.args[0]
|
||||
for call in actuator.controller.native_request.await_args_list
|
||||
if call.args[0] != "inspect_control"
|
||||
]
|
||||
assert writes == [unknown_member]
|
||||
actuator.controller.tap_at.assert_not_awaited()
|
||||
actuator.controller.swipe_coords.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_normalized_touch_in_sequence_is_mapped_to_physical_pixels_exactly_once():
|
||||
actuator = actuator_with_native()
|
||||
result = await actuator.action_sequence([touch_step()])
|
||||
assert result.ok
|
||||
command, sent = actuator.controller.native_request.await_args.args
|
||||
assert command == "touch_gesture" and sent["screen_size"] == [720, 1600]
|
||||
expected = [
|
||||
{"at_ms": 0, "points": [[180, 800]]},
|
||||
{"at_ms": 150, "points": [[180, 800]]},
|
||||
{"at_ms": 400, "points": [[360, 1200]]},
|
||||
]
|
||||
assert sent["frames"] == expected
|
||||
receipt = result.metadata["sequence_results"][0]
|
||||
assert receipt["plan"]["frames"] == expected
|
||||
assert receipt["plan"]["coordinate_space"] == "pixel"
|
||||
assert receipt["args"]["coordinate_space"] == "normalized"
|
||||
assert receipt["args"]["frames"] == touch_step()["frames"]
|
||||
@@ -25,6 +25,34 @@ def delta(arguments, *, name=None, index=0):
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"name", ["pinch", "touch_gesture", "read_control", "wait_until", "scroll_to"]
|
||||
)
|
||||
def test_native_small_actions_stop_argument_inflation(name):
|
||||
guard = ToolArgumentGuard()
|
||||
guard.observe(
|
||||
SimpleNamespace(
|
||||
tool_call_chunks=[{"index": 0, "name": name, "args": "x" * SMALL_ACTION_ARGUMENT_LIMIT}]
|
||||
)
|
||||
)
|
||||
with pytest.raises(ToolArgumentOverflow) as error:
|
||||
guard.observe(SimpleNamespace(tool_call_chunks=[{"index": 0, "args": "x"}]))
|
||||
assert error.value.details["tool"] == name
|
||||
assert error.value.details["limit_chars"] == SMALL_ACTION_ARGUMENT_LIMIT
|
||||
|
||||
|
||||
def test_standalone_native_text_allows_valid_escaped_text_but_remains_bounded():
|
||||
guard = ToolArgumentGuard()
|
||||
escaped = json.dumps({"text": "中" * 65536})
|
||||
assert len(escaped) > SMALL_ACTION_ARGUMENT_LIMIT
|
||||
guard.observe(
|
||||
SimpleNamespace(tool_call_chunks=[{"index": 0, "name": "control_action", "args": escaped}])
|
||||
)
|
||||
with pytest.raises(ToolArgumentOverflow) as error:
|
||||
guard.observe(SimpleNamespace(tool_call_chunks=[{"index": 0, "args": "x" * (512 * 1024)}]))
|
||||
assert error.value.details["limit_chars"] == 512 * 1024
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def isolate(monkeypatch, tmp_path):
|
||||
llm._ENDPOINT_BREAKER._states.clear()
|
||||
|
||||
@@ -0,0 +1,209 @@
|
||||
"""Hierarchy native semantics must survive both supported observation encodings."""
|
||||
|
||||
from copy import deepcopy
|
||||
import json
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.clients.accessibility_client import normalize_helper_elements
|
||||
from artemis.clients.ui_automator_client import _parse_hierarchy_xml_to_elements
|
||||
from artemis.utils.native_control_state import informative_native_control, native_control_state
|
||||
from artemis.utils.visualization import format_minimal_list_with_elements
|
||||
|
||||
|
||||
RANGE = {"min": 0, "max": 100, "current": 37.5, "type": 1}
|
||||
ACTIONS = ["set_progress", "set_selection", "set_text"]
|
||||
|
||||
|
||||
def hierarchy_node(**overrides):
|
||||
return {
|
||||
"text": "",
|
||||
"content-desc": "",
|
||||
"class": "android.widget.SeekBar",
|
||||
"package": "com.artemis.fixture",
|
||||
"resource-id": "com.artemis.fixture:id/range",
|
||||
"bounds": "[10,20][210,80]",
|
||||
"enabled": True,
|
||||
"editable": False,
|
||||
"password": False,
|
||||
"checked": False,
|
||||
"checkable": False,
|
||||
"selected": False,
|
||||
"focused": False,
|
||||
"scrollable": False,
|
||||
"visible-to-user": True,
|
||||
"selection_start": -1,
|
||||
"selection_end": -1,
|
||||
"range": deepcopy(RANGE),
|
||||
"actions": list(ACTIONS),
|
||||
**overrides,
|
||||
}
|
||||
|
||||
|
||||
def encode_node(node, encoding):
|
||||
if encoding == "json":
|
||||
return normalize_helper_elements(json.loads(json.dumps([node])))[0]
|
||||
attributes = {}
|
||||
for key, value in node.items():
|
||||
name = key.replace("_", "-") if key in {"selection_start", "selection_end"} else key
|
||||
attributes[name] = (
|
||||
json.dumps(value, ensure_ascii=False)
|
||||
if isinstance(value, (dict, list, bool)) or value is None
|
||||
else str(value)
|
||||
)
|
||||
root = ET.Element("hierarchy")
|
||||
ET.SubElement(root, "node", attributes)
|
||||
return _parse_hierarchy_xml_to_elements(ET.tostring(root, encoding="unicode"))[0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("encoding", ["xml", "json"])
|
||||
def test_native_range_actions_flags_and_utf16_selection_map_equally_from_xml_and_json(encoding):
|
||||
state = native_control_state(
|
||||
encode_node(hierarchy_node(editable=True, selection_start=1, selection_end=3), encoding)
|
||||
)
|
||||
assert state == {
|
||||
"range": RANGE,
|
||||
"actions": ACTIONS,
|
||||
"enabled": True,
|
||||
"scrollable": False,
|
||||
"editable": True,
|
||||
"password": False,
|
||||
"checked": False,
|
||||
"checkable": False,
|
||||
"selected": False,
|
||||
"focused": False,
|
||||
"selection_start": 1,
|
||||
"selection_end": 3,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("encoding", ["xml", "json"])
|
||||
@pytest.mark.parametrize(
|
||||
"semantics",
|
||||
[
|
||||
{"range": RANGE, "actions": []},
|
||||
{"range": None, "actions": ["set_progress"]},
|
||||
{"range": None, "actions": ["click"]},
|
||||
{"range": None, "actions": [], "scrollable": True},
|
||||
],
|
||||
)
|
||||
def test_unnamed_native_controls_are_discoverable_in_the_actual_visualization(encoding, semantics):
|
||||
node = encode_node(hierarchy_node(**semantics), encoding)
|
||||
text, elements, labels = format_minimal_list_with_elements([node], 400, 800)
|
||||
assert len(elements) == 1 and labels == ["1"]
|
||||
element = elements[0]
|
||||
assert element["text"] == "(native control)"
|
||||
assert element["native_text"] == "" and element["content_desc"] == ""
|
||||
assert element["resource_id"] == "com.artemis.fixture:id/range"
|
||||
assert element["package_name"] == "com.artemis.fixture"
|
||||
assert element["class"] == "android.widget.SeekBar"
|
||||
assert element["center"] == [110, 50] and element["bounds"] == [10, 20, 210, 80]
|
||||
assert "Control: '(native control)'" in text
|
||||
assert 'package_name: "com.artemis.fixture"' in text
|
||||
if semantics["range"] is not None:
|
||||
assert element["range"] == RANGE and '"current": 37.5' in text
|
||||
if semantics["actions"]:
|
||||
assert element["actions"] == semantics["actions"]
|
||||
assert "actions: " + ",".join(semantics["actions"]) in text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("encoding", ["xml", "json"])
|
||||
def test_native_text_description_package_and_selection_preserve_exact_observed_data(encoding):
|
||||
raw = hierarchy_node(
|
||||
text=" A😀B \n\t",
|
||||
**{
|
||||
"content-desc": " description \n\t",
|
||||
"class": "android.widget.EditText",
|
||||
"package": "com.artemis.fixture.editor",
|
||||
"editable": True,
|
||||
"selection_start": 1,
|
||||
"selection_end": 3,
|
||||
},
|
||||
)
|
||||
node = encode_node(raw, encoding)
|
||||
original = deepcopy(node)
|
||||
text, elements, _ = format_minimal_list_with_elements([node], 400, 800)
|
||||
element = elements[0]
|
||||
assert element["native_text"] == raw["text"]
|
||||
assert element["input_value"] == raw["text"]
|
||||
assert element["content_desc"] == raw["content-desc"]
|
||||
assert element["package_name"] == raw["package"]
|
||||
assert (element["selection_start"], element["selection_end"]) == (1, 3)
|
||||
assert element["range"] == RANGE and element["actions"] == ACTIONS
|
||||
assert json.dumps(raw["content-desc"], ensure_ascii=False) in text
|
||||
assert node == original
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"bad_range",
|
||||
[
|
||||
None,
|
||||
[],
|
||||
"invalid json",
|
||||
"{}",
|
||||
{"min": 0, "max": 100},
|
||||
{"min": 10, "max": 0, "current": 5},
|
||||
{"min": 0, "max": 10, "current": 11},
|
||||
{"min": False, "max": 10, "current": 1},
|
||||
{"min": "0", "max": 10, "current": 1},
|
||||
{"min": 0, "max": float("inf"), "current": 1},
|
||||
{"min": 0, "max": 10, "current": float("nan")},
|
||||
],
|
||||
)
|
||||
def test_malformed_range_does_not_hide_other_valid_native_metadata(bad_range):
|
||||
assert native_control_state({"range": bad_range, "actions": ["click"], "enabled": "true"}) == {
|
||||
"actions": ["click"],
|
||||
"enabled": True,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("encoding", ["xml", "json"])
|
||||
@pytest.mark.parametrize("field", ["min", "max", "current"])
|
||||
def test_huge_range_integers_are_omitted_without_crashing_observation(encoding, field):
|
||||
value = {**RANGE, field: 10**1000}
|
||||
node = encode_node(hierarchy_node(range=value), encoding)
|
||||
state = native_control_state(node)
|
||||
assert "range" not in state and state["actions"] == ACTIONS
|
||||
_, elements, _ = format_minimal_list_with_elements([node], 400, 800)
|
||||
assert len(elements) == 1 and "range" not in elements[0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"key", ["selection_start", "selection-start", "selection_end", "selection-end"]
|
||||
)
|
||||
@pytest.mark.parametrize("bad_offset", [True, 1.5, "1.5", "--1", "²", -2, 10**1000, "9" * 100])
|
||||
def test_invalid_or_impossible_selection_offsets_are_ignored_without_raising(key, bad_offset):
|
||||
state = native_control_state({key: bad_offset, "enabled": True})
|
||||
assert state == {"enabled": True}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", [-1, 0, 3, 2**31 - 1, "-1", "0", "3", str(2**31 - 1)])
|
||||
def test_supported_native_int_selection_offsets_are_kept(value):
|
||||
assert native_control_state({"selection-start": value}) == {"selection_start": int(value)}
|
||||
|
||||
|
||||
def test_encoded_metadata_is_bounded_and_rejects_nonstring_action_names():
|
||||
state = native_control_state(
|
||||
{
|
||||
"range": " " * 8193,
|
||||
"actions": ["click", None, 7, False, "x" * 65, *["set_progress"] * 40],
|
||||
}
|
||||
)
|
||||
assert "range" not in state
|
||||
assert state["actions"] == ["click", *["set_progress"] * 27]
|
||||
assert "actions" not in native_control_state({"actions": " " * 8193})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", [None, 0, 1, "TRUE", "False", "yes", {}, []])
|
||||
def test_metadata_flags_require_actual_boolean_values_or_canonical_xml_strings(value):
|
||||
assert native_control_state({"enabled": value}) == {}
|
||||
|
||||
|
||||
def test_passive_empty_layouts_do_not_become_actionable_controls():
|
||||
state = native_control_state({"actions": ["focus", "show_on_screen"], "range": None})
|
||||
assert not informative_native_control(state)
|
||||
text, elements, labels = format_minimal_list_with_elements(
|
||||
[hierarchy_node(actions=[], range=None)], 400, 800
|
||||
)
|
||||
assert (text, elements, labels) == ("", [], [])
|
||||
Reference in new issue
Block a user