feat(android): add native controls and continuous multi-touch

This commit is contained in:
chenmingxuan committed 2026-11-02 21:33:45 +08:00
1 parent 3dd96078db
commit 4b12ae7728
70 files changed
+13105 -179

No files matched your search

+27 -2
View File
@@ -90,6 +90,10 @@ _SCREEN_CHANGING_ACTIONS: frozenset[str] = frozenset(
"click_sequence",
"action_sequence",
"drag",
"pinch",
"touch_gesture",
"control_action",
"scroll_to",
"long_press",
"input_text",
"press_key",
@@ -198,7 +202,7 @@ class McpActionExecutor:
describes the new one. Bounded re-observation waits for adjacent stable
frames and rejects a new hierarchy paired with the previous page's bitmap.
"""
if raw_name == "action_sequence":
if raw_name in {"action_sequence", "pinch", "touch_gesture", "control_action"}:
# Keep transient controls in the recorded post-image. Repeated settling
# captures can outlast the popup and replace useful evidence with a
# later closed state. This capture does not assert screen stability.
@@ -453,7 +457,14 @@ class McpActionExecutor:
return res.code not in (ActionCode.PACKAGE_NOT_FOUND, ActionCode.INVALID_ARGS)
if raw_name == "wait_for_text":
return res.code is ActionCode.TIMEOUT
if raw_name in {"drag", "action_sequence"}:
if raw_name in {
"drag",
"action_sequence",
"pinch",
"touch_gesture",
"control_action",
"scroll_to",
}:
return res.code not in {ActionCode.INVALID_ARGS, ActionCode.UNSUPPORTED}
if raw_name == "click":
return res.metadata.get("execution") in {"not_started", "unknown"}
@@ -479,6 +490,20 @@ class McpActionExecutor:
``index_elements`` overrides the list index targets resolve against
(see :meth:`execute`).
"""
from artemis.mcp.native_actions import NATIVE_ACTIONS, validate_native_action
if raw_name in NATIVE_ACTIONS:
try:
request = validate_native_action(raw_name, args, *self._screen_size())
return (
raw_name,
request.model_dump(mode="json", exclude_none=True),
None,
{"action": raw_name},
)
except ValueError as exc:
raise _ArgError(f"Error during {raw_name}: {exc}") from exc
if raw_name in {"drag", "action_sequence"}:
from artemis.config.agent import load_agent_config
from artemis.mcp.gestures import (
+6
View File
@@ -81,6 +81,12 @@ OPTIONAL_ACTIONS: frozenset[str] = frozenset(
"swipe",
"drag",
"action_sequence",
"pinch",
"touch_gesture",
"read_control",
"control_action",
"wait_until",
"scroll_to",
"press_key",
"manage_app",
"wait_for_delay",
+16 -1
View File
@@ -38,6 +38,12 @@ OPERATOR_ACTION_TO_CANONICAL: dict[str, str] = {
"swipe": "swipe",
"drag": "drag",
"action_sequence": "action_sequence",
"pinch": "pinch",
"touch_gesture": "touch_gesture",
"read_control": "read_control",
"control_action": "control_action",
"wait_until": "wait_until",
"scroll_to": "scroll_to",
"focus_and_input_text": "input_text",
"focus_and_clear_text": "focus_and_clear_text",
"erase_one_char": "erase_one_char",
@@ -90,7 +96,16 @@ def to_canonical_call(
if canonical is None:
raise ValueError(f"Unsupported action: {verb}")
if verb in {"drag", "action_sequence"}:
if verb in {
"drag",
"action_sequence",
"pinch",
"touch_gesture",
"read_control",
"control_action",
"wait_until",
"scroll_to",
}:
return canonical, dict(action_item.get("args") or {})
if verb == "tap":
+27
View File
@@ -0,0 +1,27 @@
"""Strict, shared geometry for device actions; planning never dispatches input."""
from typing import Annotated, Literal
from pydantic import BaseModel, ConfigDict, Field, StrictInt
class GestureModel(BaseModel):
model_config = ConfigDict(
extra="forbid", strict=True, allow_inf_nan=False, str_strip_whitespace=True
)
Point = Annotated[list[StrictInt], Field(min_length=2, max_length=2)]
Duration = Annotated[int, Field(strict=True, ge=1, le=10000)]
CoordinateSpace = Literal["normalized", "pixel"]
def pixel_point(point: list[int], space: CoordinateSpace, width: int, height: int) -> list[int]:
if width <= 0 or height <= 0:
raise ValueError("Screen dimensions must be positive.")
limits = [1001, 1001] if space == "normalized" else [width, height]
if any(p < 0 or p >= limit for p, limit in zip(point, limits, strict=True)):
raise ValueError(f"Point {point} is outside the {space} screen bounds.")
if space == "pixel":
return list(point)
return [min(size - 1, p * size // 1000) for p, size in zip(point, [width, height], strict=True)]
+6 -1
View File
@@ -34,7 +34,7 @@ import asyncio
from typing import Annotated, Any
from mcp.server.fastmcp import FastMCP
from mcp.types import CallToolResult, ImageContent, TextContent
from mcp.types import CallToolResult, ImageContent, TextContent, ToolAnnotations
from artemis.mcp.action_manifest import validate_actuator
from artemis.mcp.action_specs import (
@@ -87,6 +87,11 @@ def build_action_server(actuator: Actuator, name: str = "artemis_actions") -> Fa
make_wire_handler(spec, actuator, _wrap, _wrap_exception),
name=spec.name,
description=spec.wire.description,
annotations=ToolAnnotations(
readOnlyHint=True, destructiveHint=False, idempotentHint=True
)
if spec.read_only
else None,
)
# --- Internal observation tools (never declared to an LLM) -----------------------
+101 -2
View File
@@ -70,6 +70,7 @@ from pydantic import BaseModel, Field, create_model
from artemis.core.tool_declaration import ToolDeclaration
from artemis.mcp.action_types import ActionResult
from artemis.mcp.gestures import CoordinateSpace, DragCalibration, GestureStep
from artemis.mcp.native_actions import NATIVE_REQUESTS, READ_ONLY_ACTIONS
__all__ = [
"ACTION_SPECS",
@@ -159,6 +160,7 @@ class ActionSpec:
declaration: DeclarationDialect | None = None
wire: WireDialect | None = None
differences: str = ""
read_only: bool = False
# --- Shared type aliases -------------------------------------------------------------
@@ -302,7 +304,7 @@ _SPECS: tuple[ActionSpec, ...] = (
ParamSpec(
"steps",
list[GestureStep],
"Ordered UI preparation, click, wait and drag members.",
"Ordered UI preparation, click, wait, drag, pinch, touch_gesture, control_action and wait_until members. control_action members wrap their native selector/action arguments in request. Inline text is limited to 1024 characters; use a standalone control_action for long edits. Entire encoded sequence stays within the existing 16KiB argument safety bound.",
),
),
),
@@ -821,6 +823,95 @@ _SPECS: tuple[ActionSpec, ...] = (
),
)
_NATIVE_DESCRIPTIONS = {
"pinch": (
"[ACTION] Synchronized two-finger pinch/rotation on an observed canvas. "
"center and spans use coordinate_space: pixel is current screenshot pixels; normalized is 0..1000, "
"with spans measured against the SHORT screen edge to preserve angles. "
"angle_degrees starts from the horizontal axis, positive clockwise; rotation_degrees adds rotation. "
"All paths are validated before input; off-screen paths fail without clamping. "
"The span ratio is a gesture instruction, not proof of the app's resulting scale. "
"Small spans may stay below the app's recognition threshold; choose a sufficiently large safe span "
"inside the observed canvas and read the actual post-action ticks/value before recalibrating a precise drag."
),
"touch_gesture": (
"[ACTION] Continuous synchronized native touch paths with 1..5 fingers. "
"frames contain at_ms and points; times strictly increase from 0 to at most 10000ms, "
"same finger count in every frame. Equal adjacent points hold contact; later points "
"move WITHOUT lifting, enabling hold-then-drag and multi-segment gestures. "
"Use 2..32 frames, observed geometry and explicit coordinate_space. "
"All frames are validated before input. Completion reports Android gesture dispatch; "
"verify that the app actually received the input and produced the intended effect."
),
"read_control": (
"[OBSERVE] Read ONE live native control by exact AND selector copied from the current UI. "
"Use resource_id/text/content_desc/class_name and optional package_name/bounds to disambiguate. "
"Returns current range/value, supported native actions, editable selection (UTF-16 offsets), "
"identity token and visible subtree content_signature. Missing or ambiguous controls fail explicitly. "
"Custom canvas controls may expose no range/actions; then use calibrated gestures and visual verification."
),
"control_action": (
"[ACTION] Perform ONE supported native action on a freshly resolved unique control. "
"Read_control first; copy selector exactly and optionally expected_token to reject stale evidence. "
"action=set_progress requires a finite in-range value; set_selection requires selection_start/end "
"in UTF-16 code units (never split an emoji). set_text and replace_selection require text, "
"including empty text; whitespace is preserved. Other actions take no text/value/selection. "
"ime_enter invokes the advertised IME submit action. Unsupported actions fail without coordinate fallback "
"or replay. Focus or selection changes may move the target; before subsequent edits, use "
"wait_until state=stable when needed. STALE means no action started; observe again before rebuilding "
"the request. Never resend an unknown write. An accepted action is not business verification; "
"inspect the returned/current state."
),
"wait_until": (
"[OBSERVE] Bounded polling of a unique live native control, replacing blind delays. "
"state=text requires exact expected_text; value requires expected_value and optional tolerance. "
"stable requires at least two unchanged native observations for stable_ms, not pixel stability. "
"Other states inspect visible/enabled/checked/selected or absence. "
"Ambiguity, unavailable page or transport failure never counts as absence. "
"timeout_ms is at most 30000; no input is dispatched."
),
"scroll_to": (
"[ACTION] Bounded native scrolling INSIDE a uniquely observed container until its scoped target "
"becomes visible. Copy exact container and target selectors; direction is forward/backward/up/down/left/right. "
"After each scroll, container content and the scoped target's visibility/identity must remain unchanged "
"for settle_ms before success or another scroll. Stops at max_scrolls, timeout, unsupported action "
"or no content progress. "
"Retains every scroll receipt and never repeats unacknowledged input. "
"Finding a target does not click it or verify a business outcome."
),
}
def _native_spec(name: str) -> ActionSpec:
model = NATIVE_REQUESTS[name]
params = tuple(
ParamSpec(
field_name,
field.rebuild_annotation(),
field.description,
required=field.is_required(),
default=None if field.is_required() else field.default,
)
for field_name, field in sorted(
model.model_fields.items(), key=lambda pair: not pair[1].is_required()
)
)
async def bind(actuator: Any, args: dict[str, Any]) -> ActionResult:
validated = model.model_validate(args).model_dump(mode="json", exclude_none=True)
return await getattr(actuator, name)(**validated)
description = _NATIVE_DESCRIPTIONS[name]
return ActionSpec(
name=name,
operator=OperatorDialect(description, params),
wire=WireDialect(description, params, bind),
read_only=name in READ_ONLY_ACTIONS,
)
_SPECS += tuple(_native_spec(name) for name in NATIVE_REQUESTS)
#: Canonical manifest, keyed by action name, in declaration order.
ACTION_SPECS: dict[str, ActionSpec] = {spec.name: spec for spec in _SPECS}
@@ -830,6 +921,12 @@ OPERATOR_SHELL_ORDER: tuple[str, ...] = (
"click",
"action_sequence",
"drag",
"pinch",
"touch_gesture",
"read_control",
"control_action",
"wait_until",
"scroll_to",
"input_text",
"swipe",
"press_key",
@@ -865,6 +962,8 @@ def exception_prefix(action: str) -> str:
@cache
def _operator_args_model(name: str, sequence_limit: int | None = None):
"""Builds (once) the pydantic argument model for an operator shell."""
if name in NATIVE_REQUESTS:
return NATIVE_REQUESTS[name]
dialect = ACTION_SPECS[name].operator
if dialect is None:
raise ValueError(f"Action '{name}' has no operator dialect.")
@@ -947,7 +1046,7 @@ def _projected_parameters(dialect: OperatorDialect, name: str) -> dict[str, Any]
has_model(arg) for arg in get_args(annotation)
)
if any(has_model(p.annotation) for p in dialect.params):
if name in NATIVE_REQUESTS or any(has_model(p.annotation) for p in dialect.params):
# Nullable alternatives matter for endpoint-versus-calibration requests:
# models must be able to send end=null instead of inventing an endpoint.
return convert_to_openai_tool(operator_shell_tool(name))["function"]["parameters"]
+19 -3
View File
@@ -57,12 +57,28 @@ class ActionResult(BaseModel):
@property
def agent_message(self) -> str:
"""Expose gesture receipts in the model's tool response as well as the trace."""
if self.action in {"drag", "action_sequence"} and self.metadata:
"""Expose operation receipts in the model's tool response and trace."""
if (
self.action
in {
"drag",
"action_sequence",
"pinch",
"touch_gesture",
"read_control",
"control_action",
"wait_until",
"scroll_to",
}
and self.metadata
):
import json
label = "Gesture" if self.action in {"drag", "action_sequence"} else "Native"
return (
self.message + "\nGesture receipt: " + json.dumps(self.metadata, ensure_ascii=False)
self.message
+ f"\n{label} receipt: "
+ json.dumps(self.metadata, ensure_ascii=False)
)
return self.message
+116 -3
View File
@@ -112,7 +112,10 @@ class AdbActuator:
# --- Contract --------------------------------------------------------------------
def capabilities(self) -> frozenset[str]:
return DEVICE_ACTIONS
from artemis.mcp.native_actions import NATIVE_ACTIONS, native_tools_for_capabilities
native = self.controller.driver.native_capabilities
return (DEVICE_ACTIONS - NATIVE_ACTIONS) | native_tools_for_capabilities(native)
def extensions(self) -> list[ExtensionTool]:
return []
@@ -132,6 +135,90 @@ class AdbActuator:
# --- Required action -------------------------------------------------------------
async def _native_action(self, name: str, args: dict[str, Any]) -> ActionResult:
from artemis.mcp.native_actions import READ_ONLY_ACTIONS, validate_native_action
from artemis.mcp.native_executor import execute_native
try:
request = validate_native_action(name, args, *self._dims())
except ValueError as exc:
return ActionResult.failure(
name,
str(exc),
code=ActionCode.INVALID_ARGS,
metadata={"execution": "not_started", "side_effect": "none"},
)
if name not in self.capabilities():
return ActionResult.failure(
name,
"The connected backend does not advertise this native action.",
code=ActionCode.UNSUPPORTED,
metadata={"execution": "not_started", "side_effect": "none"},
)
if resource_id := getattr(request, "require_visible", None):
try:
guard = await self._visibility_guard(resource_id)
except (OSError, RuntimeError, ValueError) as exc:
return ActionResult.failure(
name,
f"Visibility check failed before input: {exc}",
metadata={"execution": "not_started", "side_effect": "none"},
)
if not guard["visible"]:
return ActionResult.failure(
name,
"Required native control is not visible; no input executed.",
code=ActionCode.TARGET_NOT_FOUND,
metadata={
"execution": "not_started",
"side_effect": "none",
"visibility_guard": guard,
},
)
receipt = await execute_native(name, request, *self._dims(), self.controller.native_request)
receipt["read_only"] = name in READ_ONLY_ACTIONS
receipt["side_effect"] = (
"none"
if name in READ_ONLY_ACTIONS or receipt.get("execution") in {"not_started", "observed"}
else "possible"
)
if receipt.get("success"):
return ActionResult.success(
name,
"Native operation completed. Inspect the receipt and current app state; business verification remains separate.",
metadata=receipt,
)
codes = {
"UNSUPPORTED": ActionCode.UNSUPPORTED,
"NOT_FOUND": ActionCode.TARGET_NOT_FOUND,
"INVALID_ARGUMENT": ActionCode.INVALID_ARGS,
"TIMEOUT": ActionCode.TIMEOUT,
}
return ActionResult.failure(
name,
str(receipt.get("error") or receipt.get("code") or "Native action failed."),
code=codes.get(receipt.get("code"), ActionCode.DEVICE_ERROR),
metadata=receipt,
)
async def pinch(self, **kwargs: Any) -> ActionResult:
return await self._native_action("pinch", kwargs)
async def touch_gesture(self, **kwargs: Any) -> ActionResult:
return await self._native_action("touch_gesture", kwargs)
async def read_control(self, **kwargs: Any) -> ActionResult:
return await self._native_action("read_control", kwargs)
async def control_action(self, **kwargs: Any) -> ActionResult:
return await self._native_action("control_action", kwargs)
async def wait_until(self, **kwargs: Any) -> ActionResult:
return await self._native_action("wait_until", kwargs)
async def scroll_to(self, **kwargs: Any) -> ActionResult:
return await self._native_action("scroll_to", kwargs)
async def drag(self, **kwargs: Any) -> ActionResult:
"""Dispatch exact pixel geometry without a normalized-coordinate round trip."""
from artemis.mcp.gestures import DragRequest, plan_drag
@@ -305,6 +392,20 @@ class AdbActuator:
)
except ValueError as exc:
return ActionResult.failure("action_sequence", str(exc), code=ActionCode.INVALID_ARGS)
from artemis.mcp.native_actions import NATIVE_ACTIONS
unavailable = {
step["action"]
for step in planned
if step["action"] in NATIVE_ACTIONS and step["action"] not in self.capabilities()
}
if unavailable:
return ActionResult.failure(
"action_sequence",
f"Unsupported members {sorted(unavailable)}; no input executed.",
code=ActionCode.UNSUPPORTED,
metadata={"execution": "not_started", "side_effect": "none"},
)
started = time.monotonic()
receipts = [
{**step, "index": i + 1, "execution": "not_started"} for i, step in enumerate(planned)
@@ -378,6 +479,14 @@ class AdbActuator:
*receipt["coordinates"], **repeats
)
error = getattr(result, "error", None)
elif step["action"] in NATIVE_ACTIONS:
result = await self._native_action(step["action"], step["args"])
error = None if result.ok else result.message
receipt.update(result.metadata)
receipt["native_execution"] = result.metadata.get("execution", "unknown")
if not result.ok:
member_execution = result.metadata.get("execution", "unknown")
failed_code = result.code
elif step["action"] == "drag":
result = await self._dispatch_drag(step)
error = None if result.ok else result.message
@@ -392,7 +501,9 @@ class AdbActuator:
receipt.update(
execution=(
member_execution
if member_execution in {"skipped", "not_started"} or not error
if member_execution
in {"skipped", "not_started", "cancelled", "observed"}
or not error
else "unknown"
),
error=error,
@@ -418,7 +529,9 @@ class AdbActuator:
completed = sum(r["execution"] == "completed" for r in receipts)
skipped = sum(r["execution"] == "skipped" for r in receipts)
attempted = any(
r["action"] != "wait" and r["execution"] in {"completed", "unknown"} for r in receipts
r["action"] not in {"wait", "wait_until"}
and r["execution"] in {"completed", "unknown", "cancelled"}
for r in receipts
)
execution = "completed" if not failed else ("partial" if completed else "unknown")
if failed and not attempted:
+12
View File
@@ -93,6 +93,18 @@ class Actuator(Protocol):
async def drag(self, **kwargs: Any) -> ActionResult: ...
async def pinch(self, **kwargs: Any) -> ActionResult: ...
async def touch_gesture(self, **kwargs: Any) -> ActionResult: ...
async def read_control(self, **kwargs: Any) -> ActionResult: ...
async def control_action(self, **kwargs: Any) -> ActionResult: ...
async def wait_until(self, **kwargs: Any) -> ActionResult: ...
async def scroll_to(self, **kwargs: Any) -> ActionResult: ...
async def action_sequence(self, steps: list[Any]) -> ActionResult: ...
async def manage_app(self, action: str, app_name: str) -> ActionResult: ...
+81 -25
View File
@@ -7,18 +7,23 @@ values are supplied from observations by the caller, never inferred by the drive
import math
from typing import Annotated, Literal
from pydantic import BaseModel, ConfigDict, Field, StrictInt, model_validator
from pydantic import Field, StringConstraints, model_validator
class GestureModel(BaseModel):
model_config = ConfigDict(
extra="forbid", strict=True, allow_inf_nan=False, str_strip_whitespace=True
)
Point = Annotated[list[StrictInt], Field(min_length=2, max_length=2)]
Duration = Annotated[int, Field(strict=True, ge=1, le=10000)]
CoordinateSpace = Literal["normalized", "pixel"]
from artemis.mcp.action_parameters import (
CoordinateSpace,
Duration,
GestureModel,
Point,
pixel_point,
)
from artemis.mcp.native_actions import (
ControlActionRequest,
PinchRequest,
TouchGestureRequest,
WaitUntilRequest,
plan_pinch,
plan_touch,
)
class ScaleAnchor(GestureModel):
@@ -54,6 +59,7 @@ class DragRequest(GestureModel):
require_visible: str | None = Field(
default=None,
min_length=1,
max_length=512,
description="Exact previously observed resource_id of a panel control. Check live visibility before dragging; refuse input if absent. Use for auto-hiding controls.",
)
@@ -68,10 +74,10 @@ class DragRequest(GestureModel):
class ClickStep(GestureModel):
action: Literal["click"]
target: Point | Annotated[str, Field(min_length=1)] = Field(
target: Point | Annotated[str, Field(min_length=1, max_length=512)] = Field(
description="Prefer an exact observed resource_id: resolve its live bounds before tapping, refusing absent/ambiguous targets. Otherwise use [x,y] in coordinate_space; never mix screenshot pixels with normalized coordinates."
)
target_description: str = Field(min_length=1)
target_description: str = Field(min_length=1, max_length=512)
coordinate_space: CoordinateSpace = "normalized"
times: int = Field(
default=1,
@@ -84,6 +90,7 @@ class ClickStep(GestureModel):
unless_visible: str | None = Field(
default=None,
min_length=1,
max_length=512,
description="Exact previously observed resource_id inside a toggled panel (not the opener). Skip this opener click when that control is already visible; otherwise click. Avoid closing a still-open panel.",
)
@@ -101,10 +108,12 @@ class EnsureVisibleStep(GestureModel):
action: Literal["ensure_visible"]
target: str = Field(
min_length=1,
max_length=512,
description="Panel control resource_id copied VERBATIM from observed elements, including the ':id/' separator. Never rewrite it as ':id:' or guess an ID.",
)
opener: str = Field(
min_length=1,
max_length=512,
description="UI opener resource_id copied VERBATIM from observed elements, including ':id/' (not ':id:'). Tap ONCE only if target is absent; never use a business submission as an opener.",
)
target_description: str = Field(min_length=1)
@@ -125,7 +134,38 @@ class EnsureVisibleStep(GestureModel):
return self
GestureStep = ClickStep | DragStep | WaitStep | EnsureVisibleStep
class PinchStep(PinchRequest):
action: Literal["pinch"]
class TouchGestureStep(TouchGestureRequest):
action: Literal["touch_gesture"]
class SequenceControlRequest(ControlActionRequest):
text: Annotated[str, StringConstraints(strip_whitespace=False, max_length=1024)] | None = None
class ControlActionStep(GestureModel):
action: Literal["control_action"]
request: SequenceControlRequest
class WaitUntilStep(WaitUntilRequest):
action: Literal["wait_until"]
timeout_ms: int = Field(default=1500, strict=True, ge=1, le=3000)
GestureStep = (
ClickStep
| DragStep
| WaitStep
| EnsureVisibleStep
| PinchStep
| TouchGestureStep
| ControlActionStep
| WaitUntilStep
)
class SequenceRequest(GestureModel):
@@ -143,17 +183,6 @@ def validate_sequence(payload: dict, limit: int) -> SequenceRequest:
return SequenceRequest.model_validate(payload)
def pixel_point(point: list[int], space: CoordinateSpace, width: int, height: int) -> list[int]:
if width <= 0 or height <= 0:
raise ValueError("Screen dimensions must be positive.")
limits = [1001, 1001] if space == "normalized" else [width, height]
if any(p < 0 or p >= limit for p, limit in zip(point, limits, strict=True)):
raise ValueError(f"Point {point} is outside the {space} screen bounds.")
if space == "pixel":
return list(point)
return [min(size - 1, p * size // 1000) for p, size in zip(point, [width, height], strict=True)]
def plan_drag(request: DragRequest, width: int, height: int) -> dict:
start = pixel_point(request.start, request.coordinate_space, width, height)
calibration = None
@@ -233,6 +262,33 @@ def plan_sequence(request: SequenceRequest, width: int, height: int, limit: int)
for step in request.steps:
if isinstance(step, DragStep):
planned.append(plan_drag(step, width, height))
elif isinstance(step, (PinchStep, TouchGestureStep)):
geometry = (
plan_pinch(step, width, height)
if isinstance(step, PinchStep)
else plan_touch(step, width, height)
)
planned.append(
{
"action": step.action,
"args": step.model_dump(mode="json", exclude={"action"}, exclude_none=True),
"plan": geometry,
}
)
elif isinstance(step, ControlActionStep):
planned.append(
{
"action": step.action,
"args": step.request.model_dump(mode="json", exclude_none=True),
}
)
elif isinstance(step, WaitUntilStep):
planned.append(
{
"action": step.action,
"args": step.model_dump(mode="json", exclude={"action"}, exclude_none=True),
}
)
elif isinstance(step, ClickStep):
planned.append(
{
+293
View File
@@ -0,0 +1,293 @@
"""Validated native actions shared by Flash, Pro and the in-process action server."""
import math
from typing import Annotated, Literal
from pydantic import ConfigDict, Field, StrictInt, StringConstraints, model_validator
from artemis.mcp.action_parameters import (
CoordinateSpace,
Duration,
GestureModel,
Point,
pixel_point,
)
ShortText = Annotated[str, Field(min_length=1, max_length=512)]
Bounds = Annotated[list[StrictInt], Field(min_length=4, max_length=4)]
class ControlSelector(GestureModel):
"""Every supplied attribute is an exact AND match against a fresh native node."""
model_config = ConfigDict(str_strip_whitespace=False)
resource_id: ShortText | None = None
text: Annotated[str, StringConstraints(strip_whitespace=False, max_length=4096)] | None = None
content_desc: ShortText | None = None
class_name: ShortText | None = None
package_name: ShortText | None = None
bounds: Bounds | None = None
@model_validator(mode="after")
def meaningful_selector(self):
if not any((self.resource_id, self.text, self.content_desc, self.class_name)):
raise ValueError("Copy at least one observed resource_id/text/content_desc/class_name.")
if self.bounds is not None:
left, top, right, bottom = self.bounds
if right <= left or bottom <= top:
raise ValueError("Bounds must be a nonempty [left, top, right, bottom] rectangle.")
return self
class ReadControlRequest(GestureModel):
selector: ControlSelector
ControlAction = Literal[
"click",
"long_click",
"focus",
"show_on_screen",
"set_progress",
"scroll_forward",
"scroll_backward",
"scroll_up",
"scroll_down",
"scroll_left",
"scroll_right",
"set_selection",
"replace_selection",
"set_text",
"ime_enter",
]
class ControlActionRequest(ReadControlRequest):
action: ControlAction
expected_token: Annotated[str, Field(pattern=r"^[a-f0-9]{64}$")] | None = None
value: float | None = None
# Deliberately do not strip user text: leading/trailing whitespace is input data.
text: Annotated[str, StringConstraints(strip_whitespace=False, max_length=65536)] | None = None
selection_start: Annotated[int, Field(strict=True, ge=0)] | None = None
selection_end: Annotated[int, Field(strict=True, ge=0)] | None = None
@model_validator(mode="after")
def action_arguments(self):
if (self.value is not None) != (self.action == "set_progress"):
raise ValueError("Only set_progress requires a finite value.")
if (self.text is not None) != (self.action in {"set_text", "replace_selection"}):
raise ValueError(
"Only set_text/replace_selection require text (empty text is allowed)."
)
selection = self.action == "set_selection"
if (self.selection_start is not None) != selection or (
self.selection_end is not None
) != selection:
raise ValueError(
"Only set_selection requires selection_start and selection_end (UTF-16)."
)
if (
self.selection_start is not None
and self.selection_end is not None
and self.selection_end < self.selection_start
):
raise ValueError("selection_end must not precede selection_start.")
return self
class TouchFrame(GestureModel):
at_ms: Annotated[int, Field(strict=True, ge=0, le=10000)]
points: Annotated[list[Point], Field(min_length=1, max_length=5)]
class TouchGestureRequest(GestureModel):
frames: Annotated[list[TouchFrame], Field(min_length=2, max_length=32)]
coordinate_space: CoordinateSpace = "normalized"
target_description: ShortText
require_visible: ShortText | None = None
@model_validator(mode="after")
def continuous_timeline(self):
if self.frames[0].at_ms != 0:
raise ValueError("The first frame must start at 0ms.")
count = len(self.frames[0].points)
for before, after in zip(self.frames, self.frames[1:]):
if after.at_ms <= before.at_ms:
raise ValueError("Frame times must strictly increase.")
if len(after.points) != count:
raise ValueError("Keep the same fingers down throughout all frames.")
return self
class PinchRequest(GestureModel):
center: Point
start_span: Annotated[int, Field(strict=True, gt=0, le=20000)]
end_span: Annotated[int, Field(strict=True, gt=0, le=20000)]
angle_degrees: Annotated[float, Field(ge=-360, le=360)] = 0
rotation_degrees: Annotated[float, Field(ge=-360, le=360)] = 0
duration_ms: Duration = 800
coordinate_space: CoordinateSpace = "normalized"
target_description: ShortText
require_visible: ShortText | None = None
@model_validator(mode="after")
def meaningful_pinch(self):
if self.start_span == self.end_span and self.rotation_degrees == 0:
raise ValueError("Change the span or rotation; an unchanged pinch is not a zoom.")
return self
class WaitUntilRequest(ReadControlRequest):
state: Literal[
"visible",
"absent",
"enabled",
"disabled",
"checked",
"unchecked",
"selected",
"unselected",
"text",
"value",
"stable",
]
expected_text: (
Annotated[str, StringConstraints(strip_whitespace=False, max_length=4096)] | None
) = None
expected_value: float | None = None
tolerance: Annotated[float, Field(ge=0)] = 0
stable_ms: Annotated[int, Field(strict=True, ge=0, le=5000)] = 300
timeout_ms: Annotated[int, Field(strict=True, ge=1, le=30000)] = 5000
poll_interval_ms: Annotated[int, Field(strict=True, ge=50, le=1000)] = 200
@model_validator(mode="after")
def expected_arguments(self):
if (self.expected_text is not None) != (self.state == "text"):
raise ValueError("Only state=text requires expected_text (exact match).")
if (self.expected_value is not None) != (self.state == "value"):
raise ValueError("Only state=value requires expected_value.")
if self.state != "value" and self.tolerance != 0:
raise ValueError("tolerance only applies to state=value.")
if self.state == "stable" and self.stable_ms > self.timeout_ms:
raise ValueError("stable_ms must fit within timeout_ms.")
return self
class ScrollToRequest(GestureModel):
container: ControlSelector
target: ControlSelector
direction: Literal["forward", "backward", "up", "down", "left", "right"] = "forward"
max_scrolls: Annotated[int, Field(strict=True, ge=1, le=20)] = 10
timeout_ms: Annotated[int, Field(strict=True, ge=1, le=30000)] = 10000
settle_ms: Annotated[int, Field(strict=True, ge=50, le=3000)] = 300
@model_validator(mode="after")
def settle_fits(self):
if self.settle_ms > self.timeout_ms:
raise ValueError("settle_ms must fit within timeout_ms.")
return self
NATIVE_REQUESTS: dict[str, type[GestureModel]] = {
"pinch": PinchRequest,
"touch_gesture": TouchGestureRequest,
"read_control": ReadControlRequest,
"control_action": ControlActionRequest,
"wait_until": WaitUntilRequest,
"scroll_to": ScrollToRequest,
}
NATIVE_ACTIONS = frozenset(NATIVE_REQUESTS)
READ_ONLY_ACTIONS = frozenset({"read_control", "wait_until"})
def native_tools_for_capabilities(capabilities: frozenset[str]) -> frozenset[str]:
tools: set[str] = set()
if "touch_gesture" in capabilities:
tools.update({"pinch", "touch_gesture"})
if "inspect_control" in capabilities:
tools.update({"read_control", "wait_until"})
if "control_action" in capabilities:
tools.update({"control_action", "scroll_to"})
return frozenset(tools)
def plan_touch(request: TouchGestureRequest, width: int, height: int) -> dict:
frames = [
{
"at_ms": frame.at_ms,
"points": [
pixel_point(p, request.coordinate_space, width, height) for p in frame.points
],
}
for frame in request.frames
]
# Reject trajectories that merge two fingers, including between keyframes.
for before, after in zip(frames, frames[1:]):
for first in range(len(before["points"])):
for second in range(first):
a = [before["points"][first][k] - before["points"][second][k] for k in (0, 1)]
b = [after["points"][first][k] - after["points"][second][k] - a[k] for k in (0, 1)]
length = sum(v * v for v in b)
t = max(0, min(1, -sum(a[k] * b[k] for k in (0, 1)) / length)) if length else 0
if sum((a[k] + t * b[k]) ** 2 for k in (0, 1)) < 1:
raise ValueError(
"Finger paths must remain distinct; use rotation keyframes instead of crossing fingers."
)
return {
"action": "touch_gesture",
"frames": frames,
"coordinate_space": "pixel",
"screen_size": [width, height],
"target_description": request.target_description,
**({"require_visible": request.require_visible} if request.require_visible else {}),
}
def plan_pinch(request: PinchRequest, width: int, height: int) -> dict:
# A span in normalized space uses the short display edge, preserving angles/aspect.
center = pixel_point(request.center, request.coordinate_space, width, height)
factor = min(width, height) / 1000 if request.coordinate_space == "normalized" else 1
segments = max(1, math.ceil(abs(request.rotation_degrees) / 15))
if request.duration_ms < segments:
raise ValueError("duration_ms is too short for the requested rotation.")
frames = []
for index in range(segments + 1):
ratio = index / segments
radius = (request.start_span + (request.end_span - request.start_span) * ratio) * factor / 2
angle = math.radians(request.angle_degrees + request.rotation_degrees * ratio)
offset = [math.cos(angle) * radius, math.sin(angle) * radius]
frames.append(
{
"at_ms": round(request.duration_ms * ratio),
"points": [
[round(center[k] + sign * offset[k]) for k in (0, 1)] for sign in (-1, 1)
],
}
)
planned = plan_touch(
TouchGestureRequest(
frames=frames,
coordinate_space="pixel",
target_description=request.target_description,
require_visible=request.require_visible,
),
width,
height,
)
planned["action"] = "pinch"
planned["pinch"] = {
"start_span": request.start_span,
"end_span": request.end_span,
"rotation_degrees": request.rotation_degrees,
}
return planned
def validate_native_action(name: str, args: dict, width: int, height: int) -> GestureModel:
request = NATIVE_REQUESTS[name].model_validate(args)
if isinstance(request, PinchRequest):
plan_pinch(request, width, height)
elif isinstance(request, TouchGestureRequest):
plan_touch(request, width, height)
return request
+499
View File
@@ -0,0 +1,499 @@
"""Bounded native operations; accepted input is not a business verification."""
import asyncio
from collections.abc import Awaitable
import math
from time import monotonic
from typing import Any, Protocol
from uuid import uuid4
from pydantic import BaseModel, ValidationError
from artemis.mcp.native_actions import (
NATIVE_REQUESTS,
ControlActionRequest,
PinchRequest,
ReadControlRequest,
ScrollToRequest,
TouchGestureRequest,
WaitUntilRequest,
plan_pinch,
plan_touch,
)
from artemis.runtime.native_protocol import CONTROL_COMPLETION_SECONDS
sleep = asyncio.sleep
_EXECUTIONS = {"observed", "completed", "not_started", "unknown", "cancelled"}
_OBSERVATION_LIMIT = 64
class NativeCaller(Protocol):
def __call__(
self, command: str, args: dict, *, timeout: float | None = None
) -> Awaitable[dict]: ...
def _failure(code: str, error: str, *, execution: str = "not_started", **evidence: Any) -> dict:
return {"success": False, "execution": execution, "code": code, "error": error, **evidence}
def _result(name: str, receipt: dict, **evidence: Any) -> dict:
result = {**receipt, **evidence, "action": name}
result.setdefault("native_receipt", dict(receipt))
return result
async def _call(
native: NativeCaller, command: str, args: dict, *, timeout: float | None = None
) -> dict:
mutation = command != "inspect_control"
if timeout is not None and timeout <= 0:
return _failure("TIMEOUT", "The operation deadline expired before native dispatch.")
try:
if timeout is None:
result = await native(command, args, timeout=None)
else:
async with asyncio.timeout(timeout):
result = await native(command, args, timeout=timeout)
except asyncio.CancelledError:
raise
except (OSError, RuntimeError, ValueError) as exc:
return _failure(
"TIMEOUT" if isinstance(exc, TimeoutError) else "NATIVE_ERROR",
f"Native {command} failed: {exc}",
execution="unknown" if mutation else "not_started",
)
if (
not isinstance(result, dict)
or type(result.get("success")) is not bool
or not isinstance(result.get("execution"), str)
or result.get("execution") not in _EXECUTIONS
or not isinstance(result.get("code"), str)
or not result.get("code")
):
return _failure(
"INVALID_RESPONSE",
"The native operation did not return a structured execution receipt.",
execution="unknown" if mutation else "not_started",
native_receipt=result,
)
if result["success"] and result["execution"] != ("completed" if mutation else "observed"):
return _failure(
result["code"] if result["code"] != "OK" else "INVALID_RESPONSE",
"The native receipt does not establish completion of this operation.",
execution=result["execution"],
native_receipt=result,
)
return result
async def _inspect(native: NativeCaller, args: dict, *, timeout: float | None = None) -> dict:
receipt = await _call(native, "inspect_control", args, timeout=timeout)
if receipt["success"]:
if receipt.get("candidate_count", 1) != 1:
return _failure(
"AMBIGUOUS", "The selector did not resolve uniquely.", native_receipt=receipt
)
if not isinstance(receipt.get("node"), dict):
return _failure(
"INVALID_RESPONSE", "The native read has no node.", native_receipt=receipt
)
return receipt
def _selector(request: ReadControlRequest) -> dict:
return request.selector.model_dump(mode="json", exclude_none=True)
def _identity(node: dict) -> tuple[str, str] | None:
token, content = node.get("token"), node.get("content_signature")
if isinstance(token, str) and token and isinstance(content, str) and content:
return token, content
return None
def _range(node: dict) -> dict | None:
value = node.get("range")
if not isinstance(value, dict):
return None
values = [value.get(key) for key in ("min", "max", "current")]
if not all(
type(number) in (int, float) and -1e308 <= number <= 1e308 and math.isfinite(number)
for number in values
):
return None
if not value["min"] <= value["current"] <= value["max"]:
return None
return value
def _write_guard(node: dict, action: str, expected_token: str | None = None) -> dict | None:
token = node.get("token")
if not isinstance(token, str) or not token:
return _failure("INVALID_RESPONSE", "The node has no fresh identity token.")
if expected_token is not None and token != expected_token:
return _failure("STALE", "The node changed since the supplied observation.")
actions = node.get("actions")
if not isinstance(actions, list) or action not in actions:
return _failure(
"UNSUPPORTED", "The node does not advertise the requested canonical action."
)
if node.get("enabled") is not True:
return _failure("DISABLED", "The native node is not confirmed enabled.")
return None
def _remember(observations: list[dict], started: float, receipt: dict) -> None:
# Keep the first observation and the most recent bounded evidence window.
if len(observations) == _OBSERVATION_LIMIT:
del observations[1]
observations.append({"at_ms": round((monotonic() - started) * 1000), **receipt})
async def _pause(deadline: float, interval: float) -> None:
remaining = deadline - monotonic()
if remaining > 0:
await sleep(min(interval, remaining))
def _condition(request: WaitUntilRequest, node: dict) -> bool:
state = request.state
if state == "text":
if node.get("password") is True or not isinstance(node.get("text"), str):
raise ValueError("An exact readable native text value is required.")
return node["text"] == request.expected_text
if state == "value":
native_range = _range(node)
if native_range is None or request.expected_value is None:
raise ValueError("A finite native min/max/current range is required.")
return math.isclose(
native_range["current"], request.expected_value, rel_tol=0, abs_tol=request.tolerance
)
field, expected = {
"visible": ("visible", True),
"absent": ("visible", False),
"enabled": ("enabled", True),
"disabled": ("enabled", False),
"checked": ("checked", True),
"unchecked": ("checked", False),
"selected": ("selected", True),
"unselected": ("selected", False),
}[state]
if state in {"checked", "unchecked"} and node.get("checkable") is not True:
raise ValueError("A checked predicate requires an explicitly checkable node.")
if type(node.get(field)) is not bool:
raise ValueError(f"The native node does not expose a Boolean {field} state.")
return node[field] is expected
async def _wait(request: WaitUntilRequest, native: NativeCaller) -> dict:
started = monotonic()
deadline = started + request.timeout_ms / 1000
observations: list[dict] = []
attempts = 0
stable_key = None
stable_since = started
stable_samples = 0
last: dict = {}
while monotonic() < deadline:
remaining = deadline - monotonic()
if remaining <= 0:
break
last = await _inspect(native, {"selector": _selector(request)}, timeout=remaining)
attempts += 1
_remember(observations, started, last)
if monotonic() > deadline:
break
matched = False
if not last["success"]:
code = last.get("code")
if code not in {"NOT_FOUND", "READ_FAILED", "UNAVAILABLE"}:
return _result("wait_until", last, observations=observations, attempts=attempts)
# Window teardown can temporarily make a declared root unreadable.
# Poll only reads within the original deadline; an incomplete read
# proves neither absence nor continuity of a previous stable sample.
matched = code == "NOT_FOUND" and request.state == "absent"
stable_key, stable_samples = None, 0
elif request.state == "stable":
key = _identity(last["node"])
if key is None:
return _result(
"wait_until",
_failure(
"INVALID_RESPONSE", "Stability needs both token and content_signature."
),
node=last["node"],
observations=observations,
attempts=attempts,
)
now = monotonic()
if key != stable_key:
stable_key, stable_since, stable_samples = key, now, 1
else:
stable_samples += 1
matched = stable_samples >= 2 and (now - stable_since) * 1000 >= request.stable_ms
else:
try:
matched = _condition(request, last["node"])
except ValueError as exc:
return _result(
"wait_until",
_failure("INVALID_STATE", str(exc)),
node=last["node"],
observations=observations,
attempts=attempts,
)
if matched:
return _result(
"wait_until",
{"success": True, "execution": "observed", "code": "OK"},
native_receipt=last,
state=request.state,
node=last.get("node"),
observations=observations,
attempts=attempts,
)
await _pause(deadline, request.poll_interval_ms / 1000)
return _result(
"wait_until",
_failure("TIMEOUT", "The requested native state was not observed within the deadline."),
native_receipt=last,
observations=observations,
attempts=attempts,
)
async def _control(request: ControlActionRequest, native: NativeCaller) -> dict:
inspection = await _inspect(native, {"selector": _selector(request)})
if not inspection["success"]:
return _result("control_action", inspection)
node = inspection["node"]
invalid = _write_guard(node, request.action, request.expected_token)
if invalid is None and request.action == "set_progress":
native_range = _range(node)
value = request.value
if (
native_range is None
or value is None
or not math.isfinite(value)
or not native_range["min"] <= value <= native_range["max"]
or (native_range.get("type") == 0 and value != int(value))
):
invalid = _failure(
"INVALID_ARGUMENT", "The value is incompatible with the live native range."
)
if invalid is not None:
return _result("control_action", invalid, preflight=inspection, node_before=node)
args = request.model_dump(mode="json", exclude_none=True)
args["expected_token"] = node["token"]
receipt = await _call(native, "control_action", args)
return _result(
"control_action",
receipt,
preflight=inspection,
node_before=receipt.get("node_before", node),
node_after=receipt.get("node_after"),
verification="pending_observation",
)
async def _scroll(request: ScrollToRequest, native: NativeCaller) -> dict:
started = monotonic()
deadline = started + request.timeout_ms / 1000
container_selector = request.container.model_dump(mode="json", exclude_none=True)
target_args = {
"selector": request.target.model_dump(mode="json", exclude_none=True),
"scope": container_selector,
}
observations: list[dict] = []
scroll_results: list[dict] = []
last_container: dict = {}
last_target: dict = {}
def finish(receipt: dict) -> dict:
result = dict(receipt)
if result.get("execution") not in {"unknown", "cancelled"} and any(
item["native_receipt"]["success"] for item in scroll_results
):
result["execution"] = "completed"
return _result(
"scroll_to",
result,
node=last_target.get("node"),
container=last_container.get("node"),
scroll_results=scroll_results,
observations=observations,
verification="pending_observation",
)
async def inspect(args: dict) -> dict:
receipt = await _inspect(native, args, timeout=max(0, deadline - monotonic()))
_remember(observations, started, receipt)
if monotonic() > deadline:
return _failure("TIMEOUT", "The native observation arrived after the search deadline.")
return receipt
def target_visible(receipt: dict) -> bool:
if receipt["success"]:
return receipt["node"].get("visible") is True
return False
def target_state(receipt: dict) -> tuple:
# A target may be outside the bounded container signature sample.
# Its own visibility and identity must independently remain quiet too.
return (True, receipt["node"].get("token")) if target_visible(receipt) else (False,)
def target_error(receipt: dict) -> dict | None:
if not receipt["success"] and receipt.get("code") != "NOT_FOUND":
return receipt
if receipt["success"] and type(receipt["node"].get("visible")) is not bool:
return _failure("INVALID_RESPONSE", "The scoped target has no native visibility state.")
return None
for _ in range(request.max_scrolls):
if monotonic() >= deadline:
break
last_container = await inspect({"selector": container_selector})
if not last_container["success"]:
return finish(last_container)
last_target = await inspect(target_args)
error = target_error(last_target)
if error is not None:
return finish(error)
if target_visible(last_target):
return finish({"success": True, "execution": "observed", "code": "OK"})
node = last_container["node"]
action = f"scroll_{request.direction}"
invalid = _write_guard(node, action)
signature = node.get("content_signature")
if invalid is not None:
return finish(invalid)
if not isinstance(signature, str) or not signature:
return finish(_failure("INVALID_RESPONSE", "The container has no content_signature."))
if monotonic() >= deadline:
break
receipt = await _call(
native,
"control_action",
{"selector": container_selector, "action": action, "expected_token": node["token"]},
timeout=CONTROL_COMPLETION_SECONDS,
)
scroll_results.append({"node_before": node, "native_receipt": receipt})
if not receipt["success"]:
if receipt.get("code") == "ACTION_REJECTED" and receipt["execution"] == "not_started":
return finish(
_failure("NO_PROGRESS", "The native container rejected further scrolling.")
)
return finish(receipt)
if monotonic() >= deadline:
break
settle_seconds = request.settle_ms / 1000
# A native scroll accepts the animation; it does not wait for layout.
# Require a quiet content interval before the next scroll, including when
# the target briefly enters the viewport during that animation. Also give
# a delayed first change a short grace rather than reissuing input.
no_progress_deadline = min(deadline, monotonic() + max(settle_seconds, 0.5))
stable_since = monotonic()
last_signature = (signature, target_state(last_target))
changed = False
while True:
if monotonic() >= deadline:
break
last_target = await inspect(target_args)
error = target_error(last_target)
if error is not None:
return finish(error)
last_container = await inspect({"selector": container_selector})
if not last_container["success"]:
return finish(last_container)
current = last_container["node"].get("content_signature")
if not isinstance(current, str) or not current:
return finish(
_failure("INVALID_RESPONSE", "The container has no content_signature.")
)
now = monotonic()
current_state = (current, target_state(last_target))
if current_state != last_signature:
changed = True
stable_since = now
last_signature = current_state
if changed and now - stable_since >= settle_seconds - 1e-9:
break
if not changed and now >= no_progress_deadline - 1e-9:
break
await _pause(deadline, 0.05)
if monotonic() >= deadline:
break
if target_visible(last_target):
return finish({"success": True, "execution": "completed", "code": "OK"})
if not changed:
return finish(
_failure("NO_PROGRESS", "The container content did not change after scrolling.")
)
code = "TIMEOUT" if monotonic() >= deadline else "MAX_SCROLLS"
return finish(
_failure(code, "The visible target was not found within the bounded scroll search.")
)
async def execute_native(
name: str,
request: BaseModel | dict,
width: int,
height: int,
call_native: NativeCaller,
) -> dict:
"""Execute one validated generic action without retrying any write.
Cancellation always propagates. Structured helper receipts remain available
even when a surrounding wait/scroll condition was not established.
"""
try:
model = NATIVE_REQUESTS.get(name)
if model is None:
raise ValueError("Unknown native action.")
args = request.model_dump(mode="json") if isinstance(request, BaseModel) else request
parsed = model.model_validate(args)
if isinstance(parsed, (PinchRequest, TouchGestureRequest)):
planned = (
plan_pinch(parsed, width, height)
if isinstance(parsed, PinchRequest)
else plan_touch(parsed, width, height)
)
else:
planned = None
except (ValidationError, TypeError, ValueError) as exc:
return _result(name, _failure("INVALID_ARGUMENT", str(exc)))
if planned is not None:
operation_id = str(uuid4())
args = {
"operation_id": operation_id,
"frames": planned["frames"],
"screen_size": planned["screen_size"],
}
if planned.get("require_visible"):
args["require_visible"] = planned["require_visible"]
receipt = await _call(
call_native, "touch_gesture", args, timeout=planned["frames"][-1]["at_ms"] / 1000 + 5
)
return _result(
name,
receipt,
operation_id=operation_id,
plan=planned,
verification="pending_observation",
)
if isinstance(parsed, WaitUntilRequest):
return await _wait(parsed, call_native)
if isinstance(parsed, ControlActionRequest):
return await _control(parsed, call_native)
if isinstance(parsed, ScrollToRequest):
return await _scroll(parsed, call_native)
if isinstance(parsed, ReadControlRequest):
receipt = await _inspect(call_native, {"selector": _selector(parsed)})
return _result(name, receipt)
return _result(
name, _failure("INVALID_ARGUMENT", "The action has no native execution handler.")
)