feat(android): add state conditions and measured gesture feedback
Add optional ensure_state, gesture region and final value feedback shared by Flash and Pro. Preserve completed inputs when verification fails and stop repeated ineffective strategies. Regenerate pinned tool schemas for the new optional arguments.
This commit is contained in:
1 parent
6a69a595ef
commit
4ea7939d9c
20 files changed
+2779
-33
No files matched your search
@@ -10,7 +10,10 @@ from artemis.mcp.action_parameters import (
|
||||
Duration,
|
||||
GestureModel,
|
||||
Point,
|
||||
Region,
|
||||
pixel_point,
|
||||
pixel_region,
|
||||
require_in_region,
|
||||
)
|
||||
|
||||
ShortText = Annotated[str, Field(min_length=1, max_length=512)]
|
||||
@@ -71,6 +74,11 @@ class ControlActionRequest(ReadControlRequest):
|
||||
text: Annotated[str, StringConstraints(strip_whitespace=False, max_length=65536)] | None = None
|
||||
selection_start: Annotated[int, Field(strict=True, ge=0)] | None = None
|
||||
selection_end: Annotated[int, Field(strict=True, ge=0)] | None = None
|
||||
ensure_state: "ControlCondition | None" = Field(
|
||||
default=None,
|
||||
description="Desired observed state: skip an already satisfied action; otherwise write ONCE and verify by read-only polling. Never infer state from an icon or missing attributes.",
|
||||
)
|
||||
verify_timeout_ms: Annotated[int, Field(strict=True, ge=1, le=3000)] = 1500
|
||||
|
||||
@model_validator(mode="after")
|
||||
def action_arguments(self):
|
||||
@@ -106,6 +114,10 @@ class TouchGestureRequest(GestureModel):
|
||||
coordinate_space: CoordinateSpace = "normalized"
|
||||
target_description: ShortText
|
||||
require_visible: ShortText | None = None
|
||||
region: Region | None = Field(
|
||||
default=None,
|
||||
description="Observed gesture surface bounds in coordinate_space; every finger path must stay inside it.",
|
||||
)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def continuous_timeline(self):
|
||||
@@ -130,6 +142,10 @@ class PinchRequest(GestureModel):
|
||||
coordinate_space: CoordinateSpace = "normalized"
|
||||
target_description: ShortText
|
||||
require_visible: ShortText | None = None
|
||||
region: Region | None = Field(
|
||||
default=None,
|
||||
description="Observed zoom surface bounds in coordinate_space; not the whole screen or date rail. Both fingers must remain inside.",
|
||||
)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def meaningful_pinch(self):
|
||||
@@ -149,6 +165,7 @@ class WaitUntilRequest(ReadControlRequest):
|
||||
"selected",
|
||||
"unselected",
|
||||
"text",
|
||||
"content_desc",
|
||||
"value",
|
||||
"stable",
|
||||
]
|
||||
@@ -163,8 +180,8 @@ class WaitUntilRequest(ReadControlRequest):
|
||||
|
||||
@model_validator(mode="after")
|
||||
def expected_arguments(self):
|
||||
if (self.expected_text is not None) != (self.state == "text"):
|
||||
raise ValueError("Only state=text requires expected_text (exact match).")
|
||||
if (self.expected_text is not None) != (self.state in {"text", "content_desc"}):
|
||||
raise ValueError("Only state=text/content_desc requires expected_text (exact match).")
|
||||
if (self.expected_value is not None) != (self.state == "value"):
|
||||
raise ValueError("Only state=value requires expected_value.")
|
||||
if self.state != "value" and self.tolerance != 0:
|
||||
@@ -174,6 +191,26 @@ class WaitUntilRequest(ReadControlRequest):
|
||||
return self
|
||||
|
||||
|
||||
class ControlCondition(GestureModel):
|
||||
state: Literal[
|
||||
"checked", "unchecked", "selected", "unselected", "text", "content_desc", "value"
|
||||
]
|
||||
expected_text: (
|
||||
Annotated[str, StringConstraints(strip_whitespace=False, max_length=4096)] | None
|
||||
) = None
|
||||
expected_value: float | None = None
|
||||
tolerance: Annotated[float, Field(ge=0)] = 0
|
||||
|
||||
@model_validator(mode="after")
|
||||
def expected_arguments(self):
|
||||
# Reuse the wait predicate contract without duplicating its semantics.
|
||||
WaitUntilRequest(selector={"class_name": "condition"}, **self.model_dump())
|
||||
return self
|
||||
|
||||
|
||||
ControlActionRequest.model_rebuild()
|
||||
|
||||
|
||||
class ScrollToRequest(GestureModel):
|
||||
container: ControlSelector
|
||||
target: ControlSelector
|
||||
@@ -222,6 +259,10 @@ def plan_touch(request: TouchGestureRequest, width: int, height: int) -> dict:
|
||||
}
|
||||
for frame in request.frames
|
||||
]
|
||||
region = None
|
||||
if request.region is not None:
|
||||
region = pixel_region(request.region, request.coordinate_space, width, height)
|
||||
require_in_region([p for frame in frames for p in frame["points"]], region)
|
||||
# Reject trajectories that merge two fingers, including between keyframes.
|
||||
for before, after in zip(frames, frames[1:]):
|
||||
for first in range(len(before["points"])):
|
||||
@@ -240,6 +281,7 @@ def plan_touch(request: TouchGestureRequest, width: int, height: int) -> dict:
|
||||
"coordinate_space": "pixel",
|
||||
"screen_size": [width, height],
|
||||
"target_description": request.target_description,
|
||||
**({"region": region} if region is not None else {}),
|
||||
**({"require_visible": request.require_visible} if request.require_visible else {}),
|
||||
}
|
||||
|
||||
@@ -271,6 +313,9 @@ def plan_pinch(request: PinchRequest, width: int, height: int) -> dict:
|
||||
coordinate_space="pixel",
|
||||
target_description=request.target_description,
|
||||
require_visible=request.require_visible,
|
||||
region=pixel_region(request.region, request.coordinate_space, width, height)
|
||||
if request.region is not None
|
||||
else None,
|
||||
),
|
||||
width,
|
||||
height,
|
||||
|
||||
Reference in new issue
Block a user