feat(android): add state conditions and measured gesture feedback

Add optional ensure_state, gesture region and final value feedback shared by Flash and Pro. Preserve completed inputs when verification fails and stop repeated ineffective strategies. Regenerate pinned tool schemas for the new optional arguments.
This commit is contained in:
chenmingxuan committed 2026-11-02 23:20:34 +08:00
1 parent 6a69a595ef
commit 4ea7939d9c
20 files changed
+2779 -33

No files matched your search

+47 -2
View File
@@ -10,7 +10,10 @@ from artemis.mcp.action_parameters import (
Duration,
GestureModel,
Point,
Region,
pixel_point,
pixel_region,
require_in_region,
)
ShortText = Annotated[str, Field(min_length=1, max_length=512)]
@@ -71,6 +74,11 @@ class ControlActionRequest(ReadControlRequest):
text: Annotated[str, StringConstraints(strip_whitespace=False, max_length=65536)] | None = None
selection_start: Annotated[int, Field(strict=True, ge=0)] | None = None
selection_end: Annotated[int, Field(strict=True, ge=0)] | None = None
ensure_state: "ControlCondition | None" = Field(
default=None,
description="Desired observed state: skip an already satisfied action; otherwise write ONCE and verify by read-only polling. Never infer state from an icon or missing attributes.",
)
verify_timeout_ms: Annotated[int, Field(strict=True, ge=1, le=3000)] = 1500
@model_validator(mode="after")
def action_arguments(self):
@@ -106,6 +114,10 @@ class TouchGestureRequest(GestureModel):
coordinate_space: CoordinateSpace = "normalized"
target_description: ShortText
require_visible: ShortText | None = None
region: Region | None = Field(
default=None,
description="Observed gesture surface bounds in coordinate_space; every finger path must stay inside it.",
)
@model_validator(mode="after")
def continuous_timeline(self):
@@ -130,6 +142,10 @@ class PinchRequest(GestureModel):
coordinate_space: CoordinateSpace = "normalized"
target_description: ShortText
require_visible: ShortText | None = None
region: Region | None = Field(
default=None,
description="Observed zoom surface bounds in coordinate_space; not the whole screen or date rail. Both fingers must remain inside.",
)
@model_validator(mode="after")
def meaningful_pinch(self):
@@ -149,6 +165,7 @@ class WaitUntilRequest(ReadControlRequest):
"selected",
"unselected",
"text",
"content_desc",
"value",
"stable",
]
@@ -163,8 +180,8 @@ class WaitUntilRequest(ReadControlRequest):
@model_validator(mode="after")
def expected_arguments(self):
if (self.expected_text is not None) != (self.state == "text"):
raise ValueError("Only state=text requires expected_text (exact match).")
if (self.expected_text is not None) != (self.state in {"text", "content_desc"}):
raise ValueError("Only state=text/content_desc requires expected_text (exact match).")
if (self.expected_value is not None) != (self.state == "value"):
raise ValueError("Only state=value requires expected_value.")
if self.state != "value" and self.tolerance != 0:
@@ -174,6 +191,26 @@ class WaitUntilRequest(ReadControlRequest):
return self
class ControlCondition(GestureModel):
state: Literal[
"checked", "unchecked", "selected", "unselected", "text", "content_desc", "value"
]
expected_text: (
Annotated[str, StringConstraints(strip_whitespace=False, max_length=4096)] | None
) = None
expected_value: float | None = None
tolerance: Annotated[float, Field(ge=0)] = 0
@model_validator(mode="after")
def expected_arguments(self):
# Reuse the wait predicate contract without duplicating its semantics.
WaitUntilRequest(selector={"class_name": "condition"}, **self.model_dump())
return self
ControlActionRequest.model_rebuild()
class ScrollToRequest(GestureModel):
container: ControlSelector
target: ControlSelector
@@ -222,6 +259,10 @@ def plan_touch(request: TouchGestureRequest, width: int, height: int) -> dict:
}
for frame in request.frames
]
region = None
if request.region is not None:
region = pixel_region(request.region, request.coordinate_space, width, height)
require_in_region([p for frame in frames for p in frame["points"]], region)
# Reject trajectories that merge two fingers, including between keyframes.
for before, after in zip(frames, frames[1:]):
for first in range(len(before["points"])):
@@ -240,6 +281,7 @@ def plan_touch(request: TouchGestureRequest, width: int, height: int) -> dict:
"coordinate_space": "pixel",
"screen_size": [width, height],
"target_description": request.target_description,
**({"region": region} if region is not None else {}),
**({"require_visible": request.require_visible} if request.require_visible else {}),
}
@@ -271,6 +313,9 @@ def plan_pinch(request: PinchRequest, width: int, height: int) -> dict:
coordinate_space="pixel",
target_description=request.target_description,
require_visible=request.require_visible,
region=pixel_region(request.region, request.coordinate_space, width, height)
if request.region is not None
else None,
),
width,
height,