feat(android): add native controls and continuous multi-touch

This commit is contained in:
chenmingxuan committed 2026-11-02 21:33:45 +08:00
1 parent 3dd96078db
commit 4b12ae7728
70 files changed
+13105 -179

No files matched your search

+23 -1
View File
@@ -195,7 +195,15 @@ class OperatorNode:
return available_device_actions(actuator) | extension_names
except Exception as e:
logger.warning(f"Failed to read actuator capabilities: {e}")
return REQUIRED_ACTIONS | OPTIONAL_ACTIONS
from artemis.mcp.native_actions import NATIVE_ACTIONS, native_tools_for_capabilities
native = getattr(
getattr(self.ctx, "ui_adb_client", None), "native_capabilities", frozenset()
)
native = native if isinstance(native, frozenset) else frozenset()
return (
(REQUIRED_ACTIONS | OPTIONAL_ACTIONS) - NATIVE_ACTIONS
) | native_tools_for_capabilities(native)
def _get_history_and_plan(self) -> tuple[list[dict], str]:
steps = []
@@ -1560,6 +1568,20 @@ class OperatorNode:
"""The recorded target semantics: observed (index) or described (coords)."""
return {k: v for k, v in el.items() if k != "center"}
from artemis.mcp.native_actions import NATIVE_ACTIONS, validate_native_action
if tool_name in NATIVE_ACTIONS:
try:
request = validate_native_action(tool_name, args, width, height)
return [
{
"action": tool_name,
"args": request.model_dump(mode="json", exclude_none=True),
}
], None
except ValueError as exc:
return [], f"Error during {tool_name}: {exc}"
if tool_name in {"drag", "action_sequence"}:
from artemis.mcp.gestures import (
DragRequest,
+6
View File
@@ -84,6 +84,12 @@ _TURN_ENDING_ORDER = (
"click",
"action_sequence",
"drag",
"pinch",
"touch_gesture",
"read_control",
"control_action",
"wait_until",
"scroll_to",
"swipe",
"input_text",
"long_press",