feat(android): add native controls and continuous multi-touch

This commit is contained in:
chenmingxuan committed 2026-11-02 21:33:45 +08:00
1 parent 3dd96078db
commit 4b12ae7728
70 files changed
+13105 -179

No files matched your search

+39 -9
View File
@@ -38,6 +38,14 @@ OPERATOR_DEVICE_TOOLS = (
"wait_for_delay",
"long_press",
)
OPERATOR_NATIVE_TOOLS = (
"pinch",
"touch_gesture",
"read_control",
"control_action",
"wait_until",
"scroll_to",
)
def _tool_refs(text: str, tool: str) -> list[str]:
@@ -88,6 +96,7 @@ def test_full_toolset_is_the_default():
@pytest.mark.parametrize(
"tool",
OPERATOR_DEVICE_TOOLS
+ OPERATOR_NATIVE_TOOLS
+ ("video_analyzer", "search_history", "replay_steps", "get_step_screenshot"),
)
def test_removed_tool_leaves_no_reference(tool):
@@ -99,6 +108,23 @@ def test_removed_tool_leaves_no_reference(tool):
assert not _tool_refs(out, tool), f"'{tool}' still referenced in {name}"
def test_removing_all_native_capabilities_leaves_no_native_names_in_any_template():
available_tools = OPERATOR_PROMPT_TOOLSET - set(OPERATOR_NATIVE_TOOLS)
for name, template in load_operator_prompts().items():
out = apply_operator_prompt_contract(template, available_tools=available_tools)
for tool in OPERATOR_NATIVE_TOOLS:
assert tool not in out, f"'{tool}' still advertised without native capability in {name}"
assert "`click`" in out and "`drag`" in out
assert (
"Physical device actions (`click`, `action_sequence`, `drag`, `input_text`, `swipe`,"
" `press_key`, `manage_app`, `wait_for_delay`, `long_press`)" in out
)
assert (
"Turn-Ending Action (`click`, `action_sequence`, `drag`, `swipe`, `input_text`,"
" `long_press`, `press_key`, `manage_app`, or `wait_for_delay`)" in out
)
def test_removed_swipe_takes_its_teaching_along():
"""Gating removes the drag/slider teaching, not just the backticked name."""
template = load_operator_prompts()["main_template"]
@@ -131,26 +157,30 @@ def test_reduced_enums_stay_well_formed():
available_tools=OPERATOR_PROMPT_TOOLSET - {"manage_app", "wait_for_delay"},
)
assert (
"Physical device actions (`click`, `action_sequence`, `drag`, `input_text`, `swipe`, `press_key`, `long_press`)"
in out
"Physical device actions (`click`, `action_sequence`, `drag`, `pinch`, `touch_gesture`,"
" `read_control`, `control_action`, `wait_until`, `scroll_to`, `input_text`, `swipe`,"
" `press_key`, `long_press`)" in out
)
assert (
"Turn-Ending Action (`click`, `action_sequence`, `drag`, `swipe`, `input_text`, `long_press`, or `press_key`)"
in out
"Turn-Ending Action (`click`, `action_sequence`, `drag`, `pinch`, `touch_gesture`,"
" `read_control`, `control_action`, `wait_until`, `scroll_to`, `swipe`, `input_text`,"
" `long_press`, or `press_key`)" in out
)
def test_full_set_enum_slots_render_verbatim():
"""With every tool present the historical enumeration wording is reproduced."""
"""Every supported tool appears in the exact physical and turn-ending order."""
template = load_operator_prompts()["main_template"]
out = apply_operator_prompt_contract(template)
assert (
"Physical device actions (`click`, `action_sequence`, `drag`, `input_text`, `swipe`, `press_key`,"
" `manage_app`, `wait_for_delay`, `long_press`)" in out
"Physical device actions (`click`, `action_sequence`, `drag`, `pinch`, `touch_gesture`,"
" `read_control`, `control_action`, `wait_until`, `scroll_to`, `input_text`, `swipe`,"
" `press_key`, `manage_app`, `wait_for_delay`, `long_press`)" in out
)
assert (
"Turn-Ending Action (`click`, `action_sequence`, `drag`, `swipe`, `input_text`, `long_press`,"
" `press_key`, `manage_app`, or `wait_for_delay`)" in out
"Turn-Ending Action (`click`, `action_sequence`, `drag`, `pinch`, `touch_gesture`,"
" `read_control`, `control_action`, `wait_until`, `scroll_to`, `swipe`, `input_text`,"
" `long_press`, `press_key`, `manage_app`, or `wait_for_delay`)" in out
)
assert "Helper/Subagent tools (`ask_explorer`, `ask_diagnoser`, `video_analyzer`)" in out
assert "ADB/task tools (`run_adb_command`, `manage_task`, `android_file`)" in out