feat(android): add native controls and continuous multi-touch

This commit is contained in:
chenmingxuan committed 2026-11-02 21:33:45 +08:00
1 parent 3dd96078db
commit 4b12ae7728
70 files changed
+13105 -179

No files matched your search

+5 -1
View File
@@ -488,7 +488,11 @@ async def run_validation_loop(node, state: State) -> dict:
# a native sequence now, without introducing captures between burst members.
# Saturated controls legitimately produce the same image hash as before.
sequence_post_image = None
if success and any(item.get("action") == "action_sequence" for item in execution):
if success and any(
item.get("action")
in {"action_sequence", "pinch", "touch_gesture", "control_action", "scroll_to"}
for item in execution
):
captured = await _capture_live_screenshot(session, "post-sequence evidence")
if captured:
sequence_post_image = _absorb_post_screenshot(ctx, state, captured)
@@ -205,6 +205,17 @@ INPUT_TEXT_TOOL = tool_declaration("input_text")
SWIPE_TOOL = tool_declaration("swipe")
DRAG_TOOL = tool_declaration("drag")
ACTION_SEQUENCE_TOOL = tool_declaration("action_sequence")
NATIVE_TOOLS = [
tool_declaration(name)
for name in (
"pinch",
"touch_gesture",
"read_control",
"control_action",
"wait_until",
"scroll_to",
)
]
PRESS_KEY_TOOL = tool_declaration("press_key")
@@ -303,6 +314,7 @@ VALIDATOR_TOOLS_DECLARATION: list[ToolDeclaration] = [
SWIPE_TOOL,
DRAG_TOOL,
ACTION_SEQUENCE_TOOL,
*NATIVE_TOOLS,
PRESS_KEY_TOOL,
READ_NOTE_TOOL,
LIST_NOTES_TOOL,