feat(android): add native controls and continuous multi-touch
This commit is contained in:
1 parent
3dd96078db
commit
4b12ae7728
70 files changed
+13105
-179
No files matched your search
@@ -90,6 +90,10 @@ _SCREEN_CHANGING_ACTIONS: frozenset[str] = frozenset(
|
||||
"click_sequence",
|
||||
"action_sequence",
|
||||
"drag",
|
||||
"pinch",
|
||||
"touch_gesture",
|
||||
"control_action",
|
||||
"scroll_to",
|
||||
"long_press",
|
||||
"input_text",
|
||||
"press_key",
|
||||
@@ -198,7 +202,7 @@ class McpActionExecutor:
|
||||
describes the new one. Bounded re-observation waits for adjacent stable
|
||||
frames and rejects a new hierarchy paired with the previous page's bitmap.
|
||||
"""
|
||||
if raw_name == "action_sequence":
|
||||
if raw_name in {"action_sequence", "pinch", "touch_gesture", "control_action"}:
|
||||
# Keep transient controls in the recorded post-image. Repeated settling
|
||||
# captures can outlast the popup and replace useful evidence with a
|
||||
# later closed state. This capture does not assert screen stability.
|
||||
@@ -453,7 +457,14 @@ class McpActionExecutor:
|
||||
return res.code not in (ActionCode.PACKAGE_NOT_FOUND, ActionCode.INVALID_ARGS)
|
||||
if raw_name == "wait_for_text":
|
||||
return res.code is ActionCode.TIMEOUT
|
||||
if raw_name in {"drag", "action_sequence"}:
|
||||
if raw_name in {
|
||||
"drag",
|
||||
"action_sequence",
|
||||
"pinch",
|
||||
"touch_gesture",
|
||||
"control_action",
|
||||
"scroll_to",
|
||||
}:
|
||||
return res.code not in {ActionCode.INVALID_ARGS, ActionCode.UNSUPPORTED}
|
||||
if raw_name == "click":
|
||||
return res.metadata.get("execution") in {"not_started", "unknown"}
|
||||
@@ -479,6 +490,20 @@ class McpActionExecutor:
|
||||
``index_elements`` overrides the list index targets resolve against
|
||||
(see :meth:`execute`).
|
||||
"""
|
||||
from artemis.mcp.native_actions import NATIVE_ACTIONS, validate_native_action
|
||||
|
||||
if raw_name in NATIVE_ACTIONS:
|
||||
try:
|
||||
request = validate_native_action(raw_name, args, *self._screen_size())
|
||||
return (
|
||||
raw_name,
|
||||
request.model_dump(mode="json", exclude_none=True),
|
||||
None,
|
||||
{"action": raw_name},
|
||||
)
|
||||
except ValueError as exc:
|
||||
raise _ArgError(f"Error during {raw_name}: {exc}") from exc
|
||||
|
||||
if raw_name in {"drag", "action_sequence"}:
|
||||
from artemis.config.agent import load_agent_config
|
||||
from artemis.mcp.gestures import (
|
||||
|
||||
@@ -81,6 +81,12 @@ OPTIONAL_ACTIONS: frozenset[str] = frozenset(
|
||||
"swipe",
|
||||
"drag",
|
||||
"action_sequence",
|
||||
"pinch",
|
||||
"touch_gesture",
|
||||
"read_control",
|
||||
"control_action",
|
||||
"wait_until",
|
||||
"scroll_to",
|
||||
"press_key",
|
||||
"manage_app",
|
||||
"wait_for_delay",
|
||||
|
||||
@@ -38,6 +38,12 @@ OPERATOR_ACTION_TO_CANONICAL: dict[str, str] = {
|
||||
"swipe": "swipe",
|
||||
"drag": "drag",
|
||||
"action_sequence": "action_sequence",
|
||||
"pinch": "pinch",
|
||||
"touch_gesture": "touch_gesture",
|
||||
"read_control": "read_control",
|
||||
"control_action": "control_action",
|
||||
"wait_until": "wait_until",
|
||||
"scroll_to": "scroll_to",
|
||||
"focus_and_input_text": "input_text",
|
||||
"focus_and_clear_text": "focus_and_clear_text",
|
||||
"erase_one_char": "erase_one_char",
|
||||
@@ -90,7 +96,16 @@ def to_canonical_call(
|
||||
if canonical is None:
|
||||
raise ValueError(f"Unsupported action: {verb}")
|
||||
|
||||
if verb in {"drag", "action_sequence"}:
|
||||
if verb in {
|
||||
"drag",
|
||||
"action_sequence",
|
||||
"pinch",
|
||||
"touch_gesture",
|
||||
"read_control",
|
||||
"control_action",
|
||||
"wait_until",
|
||||
"scroll_to",
|
||||
}:
|
||||
return canonical, dict(action_item.get("args") or {})
|
||||
|
||||
if verb == "tap":
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
"""Strict, shared geometry for device actions; planning never dispatches input."""
|
||||
|
||||
from typing import Annotated, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, StrictInt
|
||||
|
||||
|
||||
class GestureModel(BaseModel):
|
||||
model_config = ConfigDict(
|
||||
extra="forbid", strict=True, allow_inf_nan=False, str_strip_whitespace=True
|
||||
)
|
||||
|
||||
|
||||
Point = Annotated[list[StrictInt], Field(min_length=2, max_length=2)]
|
||||
Duration = Annotated[int, Field(strict=True, ge=1, le=10000)]
|
||||
CoordinateSpace = Literal["normalized", "pixel"]
|
||||
|
||||
|
||||
def pixel_point(point: list[int], space: CoordinateSpace, width: int, height: int) -> list[int]:
|
||||
if width <= 0 or height <= 0:
|
||||
raise ValueError("Screen dimensions must be positive.")
|
||||
limits = [1001, 1001] if space == "normalized" else [width, height]
|
||||
if any(p < 0 or p >= limit for p, limit in zip(point, limits, strict=True)):
|
||||
raise ValueError(f"Point {point} is outside the {space} screen bounds.")
|
||||
if space == "pixel":
|
||||
return list(point)
|
||||
return [min(size - 1, p * size // 1000) for p, size in zip(point, [width, height], strict=True)]
|
||||
@@ -34,7 +34,7 @@ import asyncio
|
||||
from typing import Annotated, Any
|
||||
|
||||
from mcp.server.fastmcp import FastMCP
|
||||
from mcp.types import CallToolResult, ImageContent, TextContent
|
||||
from mcp.types import CallToolResult, ImageContent, TextContent, ToolAnnotations
|
||||
|
||||
from artemis.mcp.action_manifest import validate_actuator
|
||||
from artemis.mcp.action_specs import (
|
||||
@@ -87,6 +87,11 @@ def build_action_server(actuator: Actuator, name: str = "artemis_actions") -> Fa
|
||||
make_wire_handler(spec, actuator, _wrap, _wrap_exception),
|
||||
name=spec.name,
|
||||
description=spec.wire.description,
|
||||
annotations=ToolAnnotations(
|
||||
readOnlyHint=True, destructiveHint=False, idempotentHint=True
|
||||
)
|
||||
if spec.read_only
|
||||
else None,
|
||||
)
|
||||
|
||||
# --- Internal observation tools (never declared to an LLM) -----------------------
|
||||
|
||||
+101
-2
@@ -70,6 +70,7 @@ from pydantic import BaseModel, Field, create_model
|
||||
from artemis.core.tool_declaration import ToolDeclaration
|
||||
from artemis.mcp.action_types import ActionResult
|
||||
from artemis.mcp.gestures import CoordinateSpace, DragCalibration, GestureStep
|
||||
from artemis.mcp.native_actions import NATIVE_REQUESTS, READ_ONLY_ACTIONS
|
||||
|
||||
__all__ = [
|
||||
"ACTION_SPECS",
|
||||
@@ -159,6 +160,7 @@ class ActionSpec:
|
||||
declaration: DeclarationDialect | None = None
|
||||
wire: WireDialect | None = None
|
||||
differences: str = ""
|
||||
read_only: bool = False
|
||||
|
||||
|
||||
# --- Shared type aliases -------------------------------------------------------------
|
||||
@@ -302,7 +304,7 @@ _SPECS: tuple[ActionSpec, ...] = (
|
||||
ParamSpec(
|
||||
"steps",
|
||||
list[GestureStep],
|
||||
"Ordered UI preparation, click, wait and drag members.",
|
||||
"Ordered UI preparation, click, wait, drag, pinch, touch_gesture, control_action and wait_until members. control_action members wrap their native selector/action arguments in request. Inline text is limited to 1024 characters; use a standalone control_action for long edits. Entire encoded sequence stays within the existing 16KiB argument safety bound.",
|
||||
),
|
||||
),
|
||||
),
|
||||
@@ -821,6 +823,95 @@ _SPECS: tuple[ActionSpec, ...] = (
|
||||
),
|
||||
)
|
||||
|
||||
_NATIVE_DESCRIPTIONS = {
|
||||
"pinch": (
|
||||
"[ACTION] Synchronized two-finger pinch/rotation on an observed canvas. "
|
||||
"center and spans use coordinate_space: pixel is current screenshot pixels; normalized is 0..1000, "
|
||||
"with spans measured against the SHORT screen edge to preserve angles. "
|
||||
"angle_degrees starts from the horizontal axis, positive clockwise; rotation_degrees adds rotation. "
|
||||
"All paths are validated before input; off-screen paths fail without clamping. "
|
||||
"The span ratio is a gesture instruction, not proof of the app's resulting scale. "
|
||||
"Small spans may stay below the app's recognition threshold; choose a sufficiently large safe span "
|
||||
"inside the observed canvas and read the actual post-action ticks/value before recalibrating a precise drag."
|
||||
),
|
||||
"touch_gesture": (
|
||||
"[ACTION] Continuous synchronized native touch paths with 1..5 fingers. "
|
||||
"frames contain at_ms and points; times strictly increase from 0 to at most 10000ms, "
|
||||
"same finger count in every frame. Equal adjacent points hold contact; later points "
|
||||
"move WITHOUT lifting, enabling hold-then-drag and multi-segment gestures. "
|
||||
"Use 2..32 frames, observed geometry and explicit coordinate_space. "
|
||||
"All frames are validated before input. Completion reports Android gesture dispatch; "
|
||||
"verify that the app actually received the input and produced the intended effect."
|
||||
),
|
||||
"read_control": (
|
||||
"[OBSERVE] Read ONE live native control by exact AND selector copied from the current UI. "
|
||||
"Use resource_id/text/content_desc/class_name and optional package_name/bounds to disambiguate. "
|
||||
"Returns current range/value, supported native actions, editable selection (UTF-16 offsets), "
|
||||
"identity token and visible subtree content_signature. Missing or ambiguous controls fail explicitly. "
|
||||
"Custom canvas controls may expose no range/actions; then use calibrated gestures and visual verification."
|
||||
),
|
||||
"control_action": (
|
||||
"[ACTION] Perform ONE supported native action on a freshly resolved unique control. "
|
||||
"Read_control first; copy selector exactly and optionally expected_token to reject stale evidence. "
|
||||
"action=set_progress requires a finite in-range value; set_selection requires selection_start/end "
|
||||
"in UTF-16 code units (never split an emoji). set_text and replace_selection require text, "
|
||||
"including empty text; whitespace is preserved. Other actions take no text/value/selection. "
|
||||
"ime_enter invokes the advertised IME submit action. Unsupported actions fail without coordinate fallback "
|
||||
"or replay. Focus or selection changes may move the target; before subsequent edits, use "
|
||||
"wait_until state=stable when needed. STALE means no action started; observe again before rebuilding "
|
||||
"the request. Never resend an unknown write. An accepted action is not business verification; "
|
||||
"inspect the returned/current state."
|
||||
),
|
||||
"wait_until": (
|
||||
"[OBSERVE] Bounded polling of a unique live native control, replacing blind delays. "
|
||||
"state=text requires exact expected_text; value requires expected_value and optional tolerance. "
|
||||
"stable requires at least two unchanged native observations for stable_ms, not pixel stability. "
|
||||
"Other states inspect visible/enabled/checked/selected or absence. "
|
||||
"Ambiguity, unavailable page or transport failure never counts as absence. "
|
||||
"timeout_ms is at most 30000; no input is dispatched."
|
||||
),
|
||||
"scroll_to": (
|
||||
"[ACTION] Bounded native scrolling INSIDE a uniquely observed container until its scoped target "
|
||||
"becomes visible. Copy exact container and target selectors; direction is forward/backward/up/down/left/right. "
|
||||
"After each scroll, container content and the scoped target's visibility/identity must remain unchanged "
|
||||
"for settle_ms before success or another scroll. Stops at max_scrolls, timeout, unsupported action "
|
||||
"or no content progress. "
|
||||
"Retains every scroll receipt and never repeats unacknowledged input. "
|
||||
"Finding a target does not click it or verify a business outcome."
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _native_spec(name: str) -> ActionSpec:
|
||||
model = NATIVE_REQUESTS[name]
|
||||
params = tuple(
|
||||
ParamSpec(
|
||||
field_name,
|
||||
field.rebuild_annotation(),
|
||||
field.description,
|
||||
required=field.is_required(),
|
||||
default=None if field.is_required() else field.default,
|
||||
)
|
||||
for field_name, field in sorted(
|
||||
model.model_fields.items(), key=lambda pair: not pair[1].is_required()
|
||||
)
|
||||
)
|
||||
|
||||
async def bind(actuator: Any, args: dict[str, Any]) -> ActionResult:
|
||||
validated = model.model_validate(args).model_dump(mode="json", exclude_none=True)
|
||||
return await getattr(actuator, name)(**validated)
|
||||
|
||||
description = _NATIVE_DESCRIPTIONS[name]
|
||||
return ActionSpec(
|
||||
name=name,
|
||||
operator=OperatorDialect(description, params),
|
||||
wire=WireDialect(description, params, bind),
|
||||
read_only=name in READ_ONLY_ACTIONS,
|
||||
)
|
||||
|
||||
|
||||
_SPECS += tuple(_native_spec(name) for name in NATIVE_REQUESTS)
|
||||
|
||||
#: Canonical manifest, keyed by action name, in declaration order.
|
||||
ACTION_SPECS: dict[str, ActionSpec] = {spec.name: spec for spec in _SPECS}
|
||||
|
||||
@@ -830,6 +921,12 @@ OPERATOR_SHELL_ORDER: tuple[str, ...] = (
|
||||
"click",
|
||||
"action_sequence",
|
||||
"drag",
|
||||
"pinch",
|
||||
"touch_gesture",
|
||||
"read_control",
|
||||
"control_action",
|
||||
"wait_until",
|
||||
"scroll_to",
|
||||
"input_text",
|
||||
"swipe",
|
||||
"press_key",
|
||||
@@ -865,6 +962,8 @@ def exception_prefix(action: str) -> str:
|
||||
@cache
|
||||
def _operator_args_model(name: str, sequence_limit: int | None = None):
|
||||
"""Builds (once) the pydantic argument model for an operator shell."""
|
||||
if name in NATIVE_REQUESTS:
|
||||
return NATIVE_REQUESTS[name]
|
||||
dialect = ACTION_SPECS[name].operator
|
||||
if dialect is None:
|
||||
raise ValueError(f"Action '{name}' has no operator dialect.")
|
||||
@@ -947,7 +1046,7 @@ def _projected_parameters(dialect: OperatorDialect, name: str) -> dict[str, Any]
|
||||
has_model(arg) for arg in get_args(annotation)
|
||||
)
|
||||
|
||||
if any(has_model(p.annotation) for p in dialect.params):
|
||||
if name in NATIVE_REQUESTS or any(has_model(p.annotation) for p in dialect.params):
|
||||
# Nullable alternatives matter for endpoint-versus-calibration requests:
|
||||
# models must be able to send end=null instead of inventing an endpoint.
|
||||
return convert_to_openai_tool(operator_shell_tool(name))["function"]["parameters"]
|
||||
|
||||
@@ -57,12 +57,28 @@ class ActionResult(BaseModel):
|
||||
|
||||
@property
|
||||
def agent_message(self) -> str:
|
||||
"""Expose gesture receipts in the model's tool response as well as the trace."""
|
||||
if self.action in {"drag", "action_sequence"} and self.metadata:
|
||||
"""Expose operation receipts in the model's tool response and trace."""
|
||||
if (
|
||||
self.action
|
||||
in {
|
||||
"drag",
|
||||
"action_sequence",
|
||||
"pinch",
|
||||
"touch_gesture",
|
||||
"read_control",
|
||||
"control_action",
|
||||
"wait_until",
|
||||
"scroll_to",
|
||||
}
|
||||
and self.metadata
|
||||
):
|
||||
import json
|
||||
|
||||
label = "Gesture" if self.action in {"drag", "action_sequence"} else "Native"
|
||||
return (
|
||||
self.message + "\nGesture receipt: " + json.dumps(self.metadata, ensure_ascii=False)
|
||||
self.message
|
||||
+ f"\n{label} receipt: "
|
||||
+ json.dumps(self.metadata, ensure_ascii=False)
|
||||
)
|
||||
return self.message
|
||||
|
||||
|
||||
@@ -112,7 +112,10 @@ class AdbActuator:
|
||||
# --- Contract --------------------------------------------------------------------
|
||||
|
||||
def capabilities(self) -> frozenset[str]:
|
||||
return DEVICE_ACTIONS
|
||||
from artemis.mcp.native_actions import NATIVE_ACTIONS, native_tools_for_capabilities
|
||||
|
||||
native = self.controller.driver.native_capabilities
|
||||
return (DEVICE_ACTIONS - NATIVE_ACTIONS) | native_tools_for_capabilities(native)
|
||||
|
||||
def extensions(self) -> list[ExtensionTool]:
|
||||
return []
|
||||
@@ -132,6 +135,90 @@ class AdbActuator:
|
||||
|
||||
# --- Required action -------------------------------------------------------------
|
||||
|
||||
async def _native_action(self, name: str, args: dict[str, Any]) -> ActionResult:
|
||||
from artemis.mcp.native_actions import READ_ONLY_ACTIONS, validate_native_action
|
||||
from artemis.mcp.native_executor import execute_native
|
||||
|
||||
try:
|
||||
request = validate_native_action(name, args, *self._dims())
|
||||
except ValueError as exc:
|
||||
return ActionResult.failure(
|
||||
name,
|
||||
str(exc),
|
||||
code=ActionCode.INVALID_ARGS,
|
||||
metadata={"execution": "not_started", "side_effect": "none"},
|
||||
)
|
||||
if name not in self.capabilities():
|
||||
return ActionResult.failure(
|
||||
name,
|
||||
"The connected backend does not advertise this native action.",
|
||||
code=ActionCode.UNSUPPORTED,
|
||||
metadata={"execution": "not_started", "side_effect": "none"},
|
||||
)
|
||||
if resource_id := getattr(request, "require_visible", None):
|
||||
try:
|
||||
guard = await self._visibility_guard(resource_id)
|
||||
except (OSError, RuntimeError, ValueError) as exc:
|
||||
return ActionResult.failure(
|
||||
name,
|
||||
f"Visibility check failed before input: {exc}",
|
||||
metadata={"execution": "not_started", "side_effect": "none"},
|
||||
)
|
||||
if not guard["visible"]:
|
||||
return ActionResult.failure(
|
||||
name,
|
||||
"Required native control is not visible; no input executed.",
|
||||
code=ActionCode.TARGET_NOT_FOUND,
|
||||
metadata={
|
||||
"execution": "not_started",
|
||||
"side_effect": "none",
|
||||
"visibility_guard": guard,
|
||||
},
|
||||
)
|
||||
receipt = await execute_native(name, request, *self._dims(), self.controller.native_request)
|
||||
receipt["read_only"] = name in READ_ONLY_ACTIONS
|
||||
receipt["side_effect"] = (
|
||||
"none"
|
||||
if name in READ_ONLY_ACTIONS or receipt.get("execution") in {"not_started", "observed"}
|
||||
else "possible"
|
||||
)
|
||||
if receipt.get("success"):
|
||||
return ActionResult.success(
|
||||
name,
|
||||
"Native operation completed. Inspect the receipt and current app state; business verification remains separate.",
|
||||
metadata=receipt,
|
||||
)
|
||||
codes = {
|
||||
"UNSUPPORTED": ActionCode.UNSUPPORTED,
|
||||
"NOT_FOUND": ActionCode.TARGET_NOT_FOUND,
|
||||
"INVALID_ARGUMENT": ActionCode.INVALID_ARGS,
|
||||
"TIMEOUT": ActionCode.TIMEOUT,
|
||||
}
|
||||
return ActionResult.failure(
|
||||
name,
|
||||
str(receipt.get("error") or receipt.get("code") or "Native action failed."),
|
||||
code=codes.get(receipt.get("code"), ActionCode.DEVICE_ERROR),
|
||||
metadata=receipt,
|
||||
)
|
||||
|
||||
async def pinch(self, **kwargs: Any) -> ActionResult:
|
||||
return await self._native_action("pinch", kwargs)
|
||||
|
||||
async def touch_gesture(self, **kwargs: Any) -> ActionResult:
|
||||
return await self._native_action("touch_gesture", kwargs)
|
||||
|
||||
async def read_control(self, **kwargs: Any) -> ActionResult:
|
||||
return await self._native_action("read_control", kwargs)
|
||||
|
||||
async def control_action(self, **kwargs: Any) -> ActionResult:
|
||||
return await self._native_action("control_action", kwargs)
|
||||
|
||||
async def wait_until(self, **kwargs: Any) -> ActionResult:
|
||||
return await self._native_action("wait_until", kwargs)
|
||||
|
||||
async def scroll_to(self, **kwargs: Any) -> ActionResult:
|
||||
return await self._native_action("scroll_to", kwargs)
|
||||
|
||||
async def drag(self, **kwargs: Any) -> ActionResult:
|
||||
"""Dispatch exact pixel geometry without a normalized-coordinate round trip."""
|
||||
from artemis.mcp.gestures import DragRequest, plan_drag
|
||||
@@ -305,6 +392,20 @@ class AdbActuator:
|
||||
)
|
||||
except ValueError as exc:
|
||||
return ActionResult.failure("action_sequence", str(exc), code=ActionCode.INVALID_ARGS)
|
||||
from artemis.mcp.native_actions import NATIVE_ACTIONS
|
||||
|
||||
unavailable = {
|
||||
step["action"]
|
||||
for step in planned
|
||||
if step["action"] in NATIVE_ACTIONS and step["action"] not in self.capabilities()
|
||||
}
|
||||
if unavailable:
|
||||
return ActionResult.failure(
|
||||
"action_sequence",
|
||||
f"Unsupported members {sorted(unavailable)}; no input executed.",
|
||||
code=ActionCode.UNSUPPORTED,
|
||||
metadata={"execution": "not_started", "side_effect": "none"},
|
||||
)
|
||||
started = time.monotonic()
|
||||
receipts = [
|
||||
{**step, "index": i + 1, "execution": "not_started"} for i, step in enumerate(planned)
|
||||
@@ -378,6 +479,14 @@ class AdbActuator:
|
||||
*receipt["coordinates"], **repeats
|
||||
)
|
||||
error = getattr(result, "error", None)
|
||||
elif step["action"] in NATIVE_ACTIONS:
|
||||
result = await self._native_action(step["action"], step["args"])
|
||||
error = None if result.ok else result.message
|
||||
receipt.update(result.metadata)
|
||||
receipt["native_execution"] = result.metadata.get("execution", "unknown")
|
||||
if not result.ok:
|
||||
member_execution = result.metadata.get("execution", "unknown")
|
||||
failed_code = result.code
|
||||
elif step["action"] == "drag":
|
||||
result = await self._dispatch_drag(step)
|
||||
error = None if result.ok else result.message
|
||||
@@ -392,7 +501,9 @@ class AdbActuator:
|
||||
receipt.update(
|
||||
execution=(
|
||||
member_execution
|
||||
if member_execution in {"skipped", "not_started"} or not error
|
||||
if member_execution
|
||||
in {"skipped", "not_started", "cancelled", "observed"}
|
||||
or not error
|
||||
else "unknown"
|
||||
),
|
||||
error=error,
|
||||
@@ -418,7 +529,9 @@ class AdbActuator:
|
||||
completed = sum(r["execution"] == "completed" for r in receipts)
|
||||
skipped = sum(r["execution"] == "skipped" for r in receipts)
|
||||
attempted = any(
|
||||
r["action"] != "wait" and r["execution"] in {"completed", "unknown"} for r in receipts
|
||||
r["action"] not in {"wait", "wait_until"}
|
||||
and r["execution"] in {"completed", "unknown", "cancelled"}
|
||||
for r in receipts
|
||||
)
|
||||
execution = "completed" if not failed else ("partial" if completed else "unknown")
|
||||
if failed and not attempted:
|
||||
|
||||
@@ -93,6 +93,18 @@ class Actuator(Protocol):
|
||||
|
||||
async def drag(self, **kwargs: Any) -> ActionResult: ...
|
||||
|
||||
async def pinch(self, **kwargs: Any) -> ActionResult: ...
|
||||
|
||||
async def touch_gesture(self, **kwargs: Any) -> ActionResult: ...
|
||||
|
||||
async def read_control(self, **kwargs: Any) -> ActionResult: ...
|
||||
|
||||
async def control_action(self, **kwargs: Any) -> ActionResult: ...
|
||||
|
||||
async def wait_until(self, **kwargs: Any) -> ActionResult: ...
|
||||
|
||||
async def scroll_to(self, **kwargs: Any) -> ActionResult: ...
|
||||
|
||||
async def action_sequence(self, steps: list[Any]) -> ActionResult: ...
|
||||
|
||||
async def manage_app(self, action: str, app_name: str) -> ActionResult: ...
|
||||
|
||||
+81
-25
@@ -7,18 +7,23 @@ values are supplied from observations by the caller, never inferred by the drive
|
||||
import math
|
||||
from typing import Annotated, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, StrictInt, model_validator
|
||||
from pydantic import Field, StringConstraints, model_validator
|
||||
|
||||
|
||||
class GestureModel(BaseModel):
|
||||
model_config = ConfigDict(
|
||||
extra="forbid", strict=True, allow_inf_nan=False, str_strip_whitespace=True
|
||||
)
|
||||
|
||||
|
||||
Point = Annotated[list[StrictInt], Field(min_length=2, max_length=2)]
|
||||
Duration = Annotated[int, Field(strict=True, ge=1, le=10000)]
|
||||
CoordinateSpace = Literal["normalized", "pixel"]
|
||||
from artemis.mcp.action_parameters import (
|
||||
CoordinateSpace,
|
||||
Duration,
|
||||
GestureModel,
|
||||
Point,
|
||||
pixel_point,
|
||||
)
|
||||
from artemis.mcp.native_actions import (
|
||||
ControlActionRequest,
|
||||
PinchRequest,
|
||||
TouchGestureRequest,
|
||||
WaitUntilRequest,
|
||||
plan_pinch,
|
||||
plan_touch,
|
||||
)
|
||||
|
||||
|
||||
class ScaleAnchor(GestureModel):
|
||||
@@ -54,6 +59,7 @@ class DragRequest(GestureModel):
|
||||
require_visible: str | None = Field(
|
||||
default=None,
|
||||
min_length=1,
|
||||
max_length=512,
|
||||
description="Exact previously observed resource_id of a panel control. Check live visibility before dragging; refuse input if absent. Use for auto-hiding controls.",
|
||||
)
|
||||
|
||||
@@ -68,10 +74,10 @@ class DragRequest(GestureModel):
|
||||
|
||||
class ClickStep(GestureModel):
|
||||
action: Literal["click"]
|
||||
target: Point | Annotated[str, Field(min_length=1)] = Field(
|
||||
target: Point | Annotated[str, Field(min_length=1, max_length=512)] = Field(
|
||||
description="Prefer an exact observed resource_id: resolve its live bounds before tapping, refusing absent/ambiguous targets. Otherwise use [x,y] in coordinate_space; never mix screenshot pixels with normalized coordinates."
|
||||
)
|
||||
target_description: str = Field(min_length=1)
|
||||
target_description: str = Field(min_length=1, max_length=512)
|
||||
coordinate_space: CoordinateSpace = "normalized"
|
||||
times: int = Field(
|
||||
default=1,
|
||||
@@ -84,6 +90,7 @@ class ClickStep(GestureModel):
|
||||
unless_visible: str | None = Field(
|
||||
default=None,
|
||||
min_length=1,
|
||||
max_length=512,
|
||||
description="Exact previously observed resource_id inside a toggled panel (not the opener). Skip this opener click when that control is already visible; otherwise click. Avoid closing a still-open panel.",
|
||||
)
|
||||
|
||||
@@ -101,10 +108,12 @@ class EnsureVisibleStep(GestureModel):
|
||||
action: Literal["ensure_visible"]
|
||||
target: str = Field(
|
||||
min_length=1,
|
||||
max_length=512,
|
||||
description="Panel control resource_id copied VERBATIM from observed elements, including the ':id/' separator. Never rewrite it as ':id:' or guess an ID.",
|
||||
)
|
||||
opener: str = Field(
|
||||
min_length=1,
|
||||
max_length=512,
|
||||
description="UI opener resource_id copied VERBATIM from observed elements, including ':id/' (not ':id:'). Tap ONCE only if target is absent; never use a business submission as an opener.",
|
||||
)
|
||||
target_description: str = Field(min_length=1)
|
||||
@@ -125,7 +134,38 @@ class EnsureVisibleStep(GestureModel):
|
||||
return self
|
||||
|
||||
|
||||
GestureStep = ClickStep | DragStep | WaitStep | EnsureVisibleStep
|
||||
class PinchStep(PinchRequest):
|
||||
action: Literal["pinch"]
|
||||
|
||||
|
||||
class TouchGestureStep(TouchGestureRequest):
|
||||
action: Literal["touch_gesture"]
|
||||
|
||||
|
||||
class SequenceControlRequest(ControlActionRequest):
|
||||
text: Annotated[str, StringConstraints(strip_whitespace=False, max_length=1024)] | None = None
|
||||
|
||||
|
||||
class ControlActionStep(GestureModel):
|
||||
action: Literal["control_action"]
|
||||
request: SequenceControlRequest
|
||||
|
||||
|
||||
class WaitUntilStep(WaitUntilRequest):
|
||||
action: Literal["wait_until"]
|
||||
timeout_ms: int = Field(default=1500, strict=True, ge=1, le=3000)
|
||||
|
||||
|
||||
GestureStep = (
|
||||
ClickStep
|
||||
| DragStep
|
||||
| WaitStep
|
||||
| EnsureVisibleStep
|
||||
| PinchStep
|
||||
| TouchGestureStep
|
||||
| ControlActionStep
|
||||
| WaitUntilStep
|
||||
)
|
||||
|
||||
|
||||
class SequenceRequest(GestureModel):
|
||||
@@ -143,17 +183,6 @@ def validate_sequence(payload: dict, limit: int) -> SequenceRequest:
|
||||
return SequenceRequest.model_validate(payload)
|
||||
|
||||
|
||||
def pixel_point(point: list[int], space: CoordinateSpace, width: int, height: int) -> list[int]:
|
||||
if width <= 0 or height <= 0:
|
||||
raise ValueError("Screen dimensions must be positive.")
|
||||
limits = [1001, 1001] if space == "normalized" else [width, height]
|
||||
if any(p < 0 or p >= limit for p, limit in zip(point, limits, strict=True)):
|
||||
raise ValueError(f"Point {point} is outside the {space} screen bounds.")
|
||||
if space == "pixel":
|
||||
return list(point)
|
||||
return [min(size - 1, p * size // 1000) for p, size in zip(point, [width, height], strict=True)]
|
||||
|
||||
|
||||
def plan_drag(request: DragRequest, width: int, height: int) -> dict:
|
||||
start = pixel_point(request.start, request.coordinate_space, width, height)
|
||||
calibration = None
|
||||
@@ -233,6 +262,33 @@ def plan_sequence(request: SequenceRequest, width: int, height: int, limit: int)
|
||||
for step in request.steps:
|
||||
if isinstance(step, DragStep):
|
||||
planned.append(plan_drag(step, width, height))
|
||||
elif isinstance(step, (PinchStep, TouchGestureStep)):
|
||||
geometry = (
|
||||
plan_pinch(step, width, height)
|
||||
if isinstance(step, PinchStep)
|
||||
else plan_touch(step, width, height)
|
||||
)
|
||||
planned.append(
|
||||
{
|
||||
"action": step.action,
|
||||
"args": step.model_dump(mode="json", exclude={"action"}, exclude_none=True),
|
||||
"plan": geometry,
|
||||
}
|
||||
)
|
||||
elif isinstance(step, ControlActionStep):
|
||||
planned.append(
|
||||
{
|
||||
"action": step.action,
|
||||
"args": step.request.model_dump(mode="json", exclude_none=True),
|
||||
}
|
||||
)
|
||||
elif isinstance(step, WaitUntilStep):
|
||||
planned.append(
|
||||
{
|
||||
"action": step.action,
|
||||
"args": step.model_dump(mode="json", exclude={"action"}, exclude_none=True),
|
||||
}
|
||||
)
|
||||
elif isinstance(step, ClickStep):
|
||||
planned.append(
|
||||
{
|
||||
|
||||
@@ -0,0 +1,293 @@
|
||||
"""Validated native actions shared by Flash, Pro and the in-process action server."""
|
||||
|
||||
import math
|
||||
from typing import Annotated, Literal
|
||||
|
||||
from pydantic import ConfigDict, Field, StrictInt, StringConstraints, model_validator
|
||||
|
||||
from artemis.mcp.action_parameters import (
|
||||
CoordinateSpace,
|
||||
Duration,
|
||||
GestureModel,
|
||||
Point,
|
||||
pixel_point,
|
||||
)
|
||||
|
||||
ShortText = Annotated[str, Field(min_length=1, max_length=512)]
|
||||
Bounds = Annotated[list[StrictInt], Field(min_length=4, max_length=4)]
|
||||
|
||||
|
||||
class ControlSelector(GestureModel):
|
||||
"""Every supplied attribute is an exact AND match against a fresh native node."""
|
||||
|
||||
model_config = ConfigDict(str_strip_whitespace=False)
|
||||
|
||||
resource_id: ShortText | None = None
|
||||
text: Annotated[str, StringConstraints(strip_whitespace=False, max_length=4096)] | None = None
|
||||
content_desc: ShortText | None = None
|
||||
class_name: ShortText | None = None
|
||||
package_name: ShortText | None = None
|
||||
bounds: Bounds | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def meaningful_selector(self):
|
||||
if not any((self.resource_id, self.text, self.content_desc, self.class_name)):
|
||||
raise ValueError("Copy at least one observed resource_id/text/content_desc/class_name.")
|
||||
if self.bounds is not None:
|
||||
left, top, right, bottom = self.bounds
|
||||
if right <= left or bottom <= top:
|
||||
raise ValueError("Bounds must be a nonempty [left, top, right, bottom] rectangle.")
|
||||
return self
|
||||
|
||||
|
||||
class ReadControlRequest(GestureModel):
|
||||
selector: ControlSelector
|
||||
|
||||
|
||||
ControlAction = Literal[
|
||||
"click",
|
||||
"long_click",
|
||||
"focus",
|
||||
"show_on_screen",
|
||||
"set_progress",
|
||||
"scroll_forward",
|
||||
"scroll_backward",
|
||||
"scroll_up",
|
||||
"scroll_down",
|
||||
"scroll_left",
|
||||
"scroll_right",
|
||||
"set_selection",
|
||||
"replace_selection",
|
||||
"set_text",
|
||||
"ime_enter",
|
||||
]
|
||||
|
||||
|
||||
class ControlActionRequest(ReadControlRequest):
|
||||
action: ControlAction
|
||||
expected_token: Annotated[str, Field(pattern=r"^[a-f0-9]{64}$")] | None = None
|
||||
value: float | None = None
|
||||
# Deliberately do not strip user text: leading/trailing whitespace is input data.
|
||||
text: Annotated[str, StringConstraints(strip_whitespace=False, max_length=65536)] | None = None
|
||||
selection_start: Annotated[int, Field(strict=True, ge=0)] | None = None
|
||||
selection_end: Annotated[int, Field(strict=True, ge=0)] | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def action_arguments(self):
|
||||
if (self.value is not None) != (self.action == "set_progress"):
|
||||
raise ValueError("Only set_progress requires a finite value.")
|
||||
if (self.text is not None) != (self.action in {"set_text", "replace_selection"}):
|
||||
raise ValueError(
|
||||
"Only set_text/replace_selection require text (empty text is allowed)."
|
||||
)
|
||||
selection = self.action == "set_selection"
|
||||
if (self.selection_start is not None) != selection or (
|
||||
self.selection_end is not None
|
||||
) != selection:
|
||||
raise ValueError(
|
||||
"Only set_selection requires selection_start and selection_end (UTF-16)."
|
||||
)
|
||||
if (
|
||||
self.selection_start is not None
|
||||
and self.selection_end is not None
|
||||
and self.selection_end < self.selection_start
|
||||
):
|
||||
raise ValueError("selection_end must not precede selection_start.")
|
||||
return self
|
||||
|
||||
|
||||
class TouchFrame(GestureModel):
|
||||
at_ms: Annotated[int, Field(strict=True, ge=0, le=10000)]
|
||||
points: Annotated[list[Point], Field(min_length=1, max_length=5)]
|
||||
|
||||
|
||||
class TouchGestureRequest(GestureModel):
|
||||
frames: Annotated[list[TouchFrame], Field(min_length=2, max_length=32)]
|
||||
coordinate_space: CoordinateSpace = "normalized"
|
||||
target_description: ShortText
|
||||
require_visible: ShortText | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def continuous_timeline(self):
|
||||
if self.frames[0].at_ms != 0:
|
||||
raise ValueError("The first frame must start at 0ms.")
|
||||
count = len(self.frames[0].points)
|
||||
for before, after in zip(self.frames, self.frames[1:]):
|
||||
if after.at_ms <= before.at_ms:
|
||||
raise ValueError("Frame times must strictly increase.")
|
||||
if len(after.points) != count:
|
||||
raise ValueError("Keep the same fingers down throughout all frames.")
|
||||
return self
|
||||
|
||||
|
||||
class PinchRequest(GestureModel):
|
||||
center: Point
|
||||
start_span: Annotated[int, Field(strict=True, gt=0, le=20000)]
|
||||
end_span: Annotated[int, Field(strict=True, gt=0, le=20000)]
|
||||
angle_degrees: Annotated[float, Field(ge=-360, le=360)] = 0
|
||||
rotation_degrees: Annotated[float, Field(ge=-360, le=360)] = 0
|
||||
duration_ms: Duration = 800
|
||||
coordinate_space: CoordinateSpace = "normalized"
|
||||
target_description: ShortText
|
||||
require_visible: ShortText | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def meaningful_pinch(self):
|
||||
if self.start_span == self.end_span and self.rotation_degrees == 0:
|
||||
raise ValueError("Change the span or rotation; an unchanged pinch is not a zoom.")
|
||||
return self
|
||||
|
||||
|
||||
class WaitUntilRequest(ReadControlRequest):
|
||||
state: Literal[
|
||||
"visible",
|
||||
"absent",
|
||||
"enabled",
|
||||
"disabled",
|
||||
"checked",
|
||||
"unchecked",
|
||||
"selected",
|
||||
"unselected",
|
||||
"text",
|
||||
"value",
|
||||
"stable",
|
||||
]
|
||||
expected_text: (
|
||||
Annotated[str, StringConstraints(strip_whitespace=False, max_length=4096)] | None
|
||||
) = None
|
||||
expected_value: float | None = None
|
||||
tolerance: Annotated[float, Field(ge=0)] = 0
|
||||
stable_ms: Annotated[int, Field(strict=True, ge=0, le=5000)] = 300
|
||||
timeout_ms: Annotated[int, Field(strict=True, ge=1, le=30000)] = 5000
|
||||
poll_interval_ms: Annotated[int, Field(strict=True, ge=50, le=1000)] = 200
|
||||
|
||||
@model_validator(mode="after")
|
||||
def expected_arguments(self):
|
||||
if (self.expected_text is not None) != (self.state == "text"):
|
||||
raise ValueError("Only state=text requires expected_text (exact match).")
|
||||
if (self.expected_value is not None) != (self.state == "value"):
|
||||
raise ValueError("Only state=value requires expected_value.")
|
||||
if self.state != "value" and self.tolerance != 0:
|
||||
raise ValueError("tolerance only applies to state=value.")
|
||||
if self.state == "stable" and self.stable_ms > self.timeout_ms:
|
||||
raise ValueError("stable_ms must fit within timeout_ms.")
|
||||
return self
|
||||
|
||||
|
||||
class ScrollToRequest(GestureModel):
|
||||
container: ControlSelector
|
||||
target: ControlSelector
|
||||
direction: Literal["forward", "backward", "up", "down", "left", "right"] = "forward"
|
||||
max_scrolls: Annotated[int, Field(strict=True, ge=1, le=20)] = 10
|
||||
timeout_ms: Annotated[int, Field(strict=True, ge=1, le=30000)] = 10000
|
||||
settle_ms: Annotated[int, Field(strict=True, ge=50, le=3000)] = 300
|
||||
|
||||
@model_validator(mode="after")
|
||||
def settle_fits(self):
|
||||
if self.settle_ms > self.timeout_ms:
|
||||
raise ValueError("settle_ms must fit within timeout_ms.")
|
||||
return self
|
||||
|
||||
|
||||
NATIVE_REQUESTS: dict[str, type[GestureModel]] = {
|
||||
"pinch": PinchRequest,
|
||||
"touch_gesture": TouchGestureRequest,
|
||||
"read_control": ReadControlRequest,
|
||||
"control_action": ControlActionRequest,
|
||||
"wait_until": WaitUntilRequest,
|
||||
"scroll_to": ScrollToRequest,
|
||||
}
|
||||
NATIVE_ACTIONS = frozenset(NATIVE_REQUESTS)
|
||||
READ_ONLY_ACTIONS = frozenset({"read_control", "wait_until"})
|
||||
|
||||
|
||||
def native_tools_for_capabilities(capabilities: frozenset[str]) -> frozenset[str]:
|
||||
tools: set[str] = set()
|
||||
if "touch_gesture" in capabilities:
|
||||
tools.update({"pinch", "touch_gesture"})
|
||||
if "inspect_control" in capabilities:
|
||||
tools.update({"read_control", "wait_until"})
|
||||
if "control_action" in capabilities:
|
||||
tools.update({"control_action", "scroll_to"})
|
||||
return frozenset(tools)
|
||||
|
||||
|
||||
def plan_touch(request: TouchGestureRequest, width: int, height: int) -> dict:
|
||||
frames = [
|
||||
{
|
||||
"at_ms": frame.at_ms,
|
||||
"points": [
|
||||
pixel_point(p, request.coordinate_space, width, height) for p in frame.points
|
||||
],
|
||||
}
|
||||
for frame in request.frames
|
||||
]
|
||||
# Reject trajectories that merge two fingers, including between keyframes.
|
||||
for before, after in zip(frames, frames[1:]):
|
||||
for first in range(len(before["points"])):
|
||||
for second in range(first):
|
||||
a = [before["points"][first][k] - before["points"][second][k] for k in (0, 1)]
|
||||
b = [after["points"][first][k] - after["points"][second][k] - a[k] for k in (0, 1)]
|
||||
length = sum(v * v for v in b)
|
||||
t = max(0, min(1, -sum(a[k] * b[k] for k in (0, 1)) / length)) if length else 0
|
||||
if sum((a[k] + t * b[k]) ** 2 for k in (0, 1)) < 1:
|
||||
raise ValueError(
|
||||
"Finger paths must remain distinct; use rotation keyframes instead of crossing fingers."
|
||||
)
|
||||
return {
|
||||
"action": "touch_gesture",
|
||||
"frames": frames,
|
||||
"coordinate_space": "pixel",
|
||||
"screen_size": [width, height],
|
||||
"target_description": request.target_description,
|
||||
**({"require_visible": request.require_visible} if request.require_visible else {}),
|
||||
}
|
||||
|
||||
|
||||
def plan_pinch(request: PinchRequest, width: int, height: int) -> dict:
|
||||
# A span in normalized space uses the short display edge, preserving angles/aspect.
|
||||
center = pixel_point(request.center, request.coordinate_space, width, height)
|
||||
factor = min(width, height) / 1000 if request.coordinate_space == "normalized" else 1
|
||||
segments = max(1, math.ceil(abs(request.rotation_degrees) / 15))
|
||||
if request.duration_ms < segments:
|
||||
raise ValueError("duration_ms is too short for the requested rotation.")
|
||||
frames = []
|
||||
for index in range(segments + 1):
|
||||
ratio = index / segments
|
||||
radius = (request.start_span + (request.end_span - request.start_span) * ratio) * factor / 2
|
||||
angle = math.radians(request.angle_degrees + request.rotation_degrees * ratio)
|
||||
offset = [math.cos(angle) * radius, math.sin(angle) * radius]
|
||||
frames.append(
|
||||
{
|
||||
"at_ms": round(request.duration_ms * ratio),
|
||||
"points": [
|
||||
[round(center[k] + sign * offset[k]) for k in (0, 1)] for sign in (-1, 1)
|
||||
],
|
||||
}
|
||||
)
|
||||
planned = plan_touch(
|
||||
TouchGestureRequest(
|
||||
frames=frames,
|
||||
coordinate_space="pixel",
|
||||
target_description=request.target_description,
|
||||
require_visible=request.require_visible,
|
||||
),
|
||||
width,
|
||||
height,
|
||||
)
|
||||
planned["action"] = "pinch"
|
||||
planned["pinch"] = {
|
||||
"start_span": request.start_span,
|
||||
"end_span": request.end_span,
|
||||
"rotation_degrees": request.rotation_degrees,
|
||||
}
|
||||
return planned
|
||||
|
||||
|
||||
def validate_native_action(name: str, args: dict, width: int, height: int) -> GestureModel:
|
||||
request = NATIVE_REQUESTS[name].model_validate(args)
|
||||
if isinstance(request, PinchRequest):
|
||||
plan_pinch(request, width, height)
|
||||
elif isinstance(request, TouchGestureRequest):
|
||||
plan_touch(request, width, height)
|
||||
return request
|
||||
@@ -0,0 +1,499 @@
|
||||
"""Bounded native operations; accepted input is not a business verification."""
|
||||
|
||||
import asyncio
|
||||
from collections.abc import Awaitable
|
||||
import math
|
||||
from time import monotonic
|
||||
from typing import Any, Protocol
|
||||
from uuid import uuid4
|
||||
|
||||
from pydantic import BaseModel, ValidationError
|
||||
|
||||
from artemis.mcp.native_actions import (
|
||||
NATIVE_REQUESTS,
|
||||
ControlActionRequest,
|
||||
PinchRequest,
|
||||
ReadControlRequest,
|
||||
ScrollToRequest,
|
||||
TouchGestureRequest,
|
||||
WaitUntilRequest,
|
||||
plan_pinch,
|
||||
plan_touch,
|
||||
)
|
||||
from artemis.runtime.native_protocol import CONTROL_COMPLETION_SECONDS
|
||||
|
||||
sleep = asyncio.sleep
|
||||
_EXECUTIONS = {"observed", "completed", "not_started", "unknown", "cancelled"}
|
||||
_OBSERVATION_LIMIT = 64
|
||||
|
||||
|
||||
class NativeCaller(Protocol):
|
||||
def __call__(
|
||||
self, command: str, args: dict, *, timeout: float | None = None
|
||||
) -> Awaitable[dict]: ...
|
||||
|
||||
|
||||
def _failure(code: str, error: str, *, execution: str = "not_started", **evidence: Any) -> dict:
|
||||
return {"success": False, "execution": execution, "code": code, "error": error, **evidence}
|
||||
|
||||
|
||||
def _result(name: str, receipt: dict, **evidence: Any) -> dict:
|
||||
result = {**receipt, **evidence, "action": name}
|
||||
result.setdefault("native_receipt", dict(receipt))
|
||||
return result
|
||||
|
||||
|
||||
async def _call(
|
||||
native: NativeCaller, command: str, args: dict, *, timeout: float | None = None
|
||||
) -> dict:
|
||||
mutation = command != "inspect_control"
|
||||
if timeout is not None and timeout <= 0:
|
||||
return _failure("TIMEOUT", "The operation deadline expired before native dispatch.")
|
||||
try:
|
||||
if timeout is None:
|
||||
result = await native(command, args, timeout=None)
|
||||
else:
|
||||
async with asyncio.timeout(timeout):
|
||||
result = await native(command, args, timeout=timeout)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except (OSError, RuntimeError, ValueError) as exc:
|
||||
return _failure(
|
||||
"TIMEOUT" if isinstance(exc, TimeoutError) else "NATIVE_ERROR",
|
||||
f"Native {command} failed: {exc}",
|
||||
execution="unknown" if mutation else "not_started",
|
||||
)
|
||||
if (
|
||||
not isinstance(result, dict)
|
||||
or type(result.get("success")) is not bool
|
||||
or not isinstance(result.get("execution"), str)
|
||||
or result.get("execution") not in _EXECUTIONS
|
||||
or not isinstance(result.get("code"), str)
|
||||
or not result.get("code")
|
||||
):
|
||||
return _failure(
|
||||
"INVALID_RESPONSE",
|
||||
"The native operation did not return a structured execution receipt.",
|
||||
execution="unknown" if mutation else "not_started",
|
||||
native_receipt=result,
|
||||
)
|
||||
if result["success"] and result["execution"] != ("completed" if mutation else "observed"):
|
||||
return _failure(
|
||||
result["code"] if result["code"] != "OK" else "INVALID_RESPONSE",
|
||||
"The native receipt does not establish completion of this operation.",
|
||||
execution=result["execution"],
|
||||
native_receipt=result,
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
async def _inspect(native: NativeCaller, args: dict, *, timeout: float | None = None) -> dict:
|
||||
receipt = await _call(native, "inspect_control", args, timeout=timeout)
|
||||
if receipt["success"]:
|
||||
if receipt.get("candidate_count", 1) != 1:
|
||||
return _failure(
|
||||
"AMBIGUOUS", "The selector did not resolve uniquely.", native_receipt=receipt
|
||||
)
|
||||
if not isinstance(receipt.get("node"), dict):
|
||||
return _failure(
|
||||
"INVALID_RESPONSE", "The native read has no node.", native_receipt=receipt
|
||||
)
|
||||
return receipt
|
||||
|
||||
|
||||
def _selector(request: ReadControlRequest) -> dict:
|
||||
return request.selector.model_dump(mode="json", exclude_none=True)
|
||||
|
||||
|
||||
def _identity(node: dict) -> tuple[str, str] | None:
|
||||
token, content = node.get("token"), node.get("content_signature")
|
||||
if isinstance(token, str) and token and isinstance(content, str) and content:
|
||||
return token, content
|
||||
return None
|
||||
|
||||
|
||||
def _range(node: dict) -> dict | None:
|
||||
value = node.get("range")
|
||||
if not isinstance(value, dict):
|
||||
return None
|
||||
values = [value.get(key) for key in ("min", "max", "current")]
|
||||
if not all(
|
||||
type(number) in (int, float) and -1e308 <= number <= 1e308 and math.isfinite(number)
|
||||
for number in values
|
||||
):
|
||||
return None
|
||||
if not value["min"] <= value["current"] <= value["max"]:
|
||||
return None
|
||||
return value
|
||||
|
||||
|
||||
def _write_guard(node: dict, action: str, expected_token: str | None = None) -> dict | None:
|
||||
token = node.get("token")
|
||||
if not isinstance(token, str) or not token:
|
||||
return _failure("INVALID_RESPONSE", "The node has no fresh identity token.")
|
||||
if expected_token is not None and token != expected_token:
|
||||
return _failure("STALE", "The node changed since the supplied observation.")
|
||||
actions = node.get("actions")
|
||||
if not isinstance(actions, list) or action not in actions:
|
||||
return _failure(
|
||||
"UNSUPPORTED", "The node does not advertise the requested canonical action."
|
||||
)
|
||||
if node.get("enabled") is not True:
|
||||
return _failure("DISABLED", "The native node is not confirmed enabled.")
|
||||
return None
|
||||
|
||||
|
||||
def _remember(observations: list[dict], started: float, receipt: dict) -> None:
|
||||
# Keep the first observation and the most recent bounded evidence window.
|
||||
if len(observations) == _OBSERVATION_LIMIT:
|
||||
del observations[1]
|
||||
observations.append({"at_ms": round((monotonic() - started) * 1000), **receipt})
|
||||
|
||||
|
||||
async def _pause(deadline: float, interval: float) -> None:
|
||||
remaining = deadline - monotonic()
|
||||
if remaining > 0:
|
||||
await sleep(min(interval, remaining))
|
||||
|
||||
|
||||
def _condition(request: WaitUntilRequest, node: dict) -> bool:
|
||||
state = request.state
|
||||
if state == "text":
|
||||
if node.get("password") is True or not isinstance(node.get("text"), str):
|
||||
raise ValueError("An exact readable native text value is required.")
|
||||
return node["text"] == request.expected_text
|
||||
if state == "value":
|
||||
native_range = _range(node)
|
||||
if native_range is None or request.expected_value is None:
|
||||
raise ValueError("A finite native min/max/current range is required.")
|
||||
return math.isclose(
|
||||
native_range["current"], request.expected_value, rel_tol=0, abs_tol=request.tolerance
|
||||
)
|
||||
field, expected = {
|
||||
"visible": ("visible", True),
|
||||
"absent": ("visible", False),
|
||||
"enabled": ("enabled", True),
|
||||
"disabled": ("enabled", False),
|
||||
"checked": ("checked", True),
|
||||
"unchecked": ("checked", False),
|
||||
"selected": ("selected", True),
|
||||
"unselected": ("selected", False),
|
||||
}[state]
|
||||
if state in {"checked", "unchecked"} and node.get("checkable") is not True:
|
||||
raise ValueError("A checked predicate requires an explicitly checkable node.")
|
||||
if type(node.get(field)) is not bool:
|
||||
raise ValueError(f"The native node does not expose a Boolean {field} state.")
|
||||
return node[field] is expected
|
||||
|
||||
|
||||
async def _wait(request: WaitUntilRequest, native: NativeCaller) -> dict:
|
||||
started = monotonic()
|
||||
deadline = started + request.timeout_ms / 1000
|
||||
observations: list[dict] = []
|
||||
attempts = 0
|
||||
stable_key = None
|
||||
stable_since = started
|
||||
stable_samples = 0
|
||||
last: dict = {}
|
||||
while monotonic() < deadline:
|
||||
remaining = deadline - monotonic()
|
||||
if remaining <= 0:
|
||||
break
|
||||
last = await _inspect(native, {"selector": _selector(request)}, timeout=remaining)
|
||||
attempts += 1
|
||||
_remember(observations, started, last)
|
||||
if monotonic() > deadline:
|
||||
break
|
||||
matched = False
|
||||
if not last["success"]:
|
||||
code = last.get("code")
|
||||
if code not in {"NOT_FOUND", "READ_FAILED", "UNAVAILABLE"}:
|
||||
return _result("wait_until", last, observations=observations, attempts=attempts)
|
||||
# Window teardown can temporarily make a declared root unreadable.
|
||||
# Poll only reads within the original deadline; an incomplete read
|
||||
# proves neither absence nor continuity of a previous stable sample.
|
||||
matched = code == "NOT_FOUND" and request.state == "absent"
|
||||
stable_key, stable_samples = None, 0
|
||||
elif request.state == "stable":
|
||||
key = _identity(last["node"])
|
||||
if key is None:
|
||||
return _result(
|
||||
"wait_until",
|
||||
_failure(
|
||||
"INVALID_RESPONSE", "Stability needs both token and content_signature."
|
||||
),
|
||||
node=last["node"],
|
||||
observations=observations,
|
||||
attempts=attempts,
|
||||
)
|
||||
now = monotonic()
|
||||
if key != stable_key:
|
||||
stable_key, stable_since, stable_samples = key, now, 1
|
||||
else:
|
||||
stable_samples += 1
|
||||
matched = stable_samples >= 2 and (now - stable_since) * 1000 >= request.stable_ms
|
||||
else:
|
||||
try:
|
||||
matched = _condition(request, last["node"])
|
||||
except ValueError as exc:
|
||||
return _result(
|
||||
"wait_until",
|
||||
_failure("INVALID_STATE", str(exc)),
|
||||
node=last["node"],
|
||||
observations=observations,
|
||||
attempts=attempts,
|
||||
)
|
||||
if matched:
|
||||
return _result(
|
||||
"wait_until",
|
||||
{"success": True, "execution": "observed", "code": "OK"},
|
||||
native_receipt=last,
|
||||
state=request.state,
|
||||
node=last.get("node"),
|
||||
observations=observations,
|
||||
attempts=attempts,
|
||||
)
|
||||
await _pause(deadline, request.poll_interval_ms / 1000)
|
||||
return _result(
|
||||
"wait_until",
|
||||
_failure("TIMEOUT", "The requested native state was not observed within the deadline."),
|
||||
native_receipt=last,
|
||||
observations=observations,
|
||||
attempts=attempts,
|
||||
)
|
||||
|
||||
|
||||
async def _control(request: ControlActionRequest, native: NativeCaller) -> dict:
|
||||
inspection = await _inspect(native, {"selector": _selector(request)})
|
||||
if not inspection["success"]:
|
||||
return _result("control_action", inspection)
|
||||
node = inspection["node"]
|
||||
invalid = _write_guard(node, request.action, request.expected_token)
|
||||
if invalid is None and request.action == "set_progress":
|
||||
native_range = _range(node)
|
||||
value = request.value
|
||||
if (
|
||||
native_range is None
|
||||
or value is None
|
||||
or not math.isfinite(value)
|
||||
or not native_range["min"] <= value <= native_range["max"]
|
||||
or (native_range.get("type") == 0 and value != int(value))
|
||||
):
|
||||
invalid = _failure(
|
||||
"INVALID_ARGUMENT", "The value is incompatible with the live native range."
|
||||
)
|
||||
if invalid is not None:
|
||||
return _result("control_action", invalid, preflight=inspection, node_before=node)
|
||||
args = request.model_dump(mode="json", exclude_none=True)
|
||||
args["expected_token"] = node["token"]
|
||||
receipt = await _call(native, "control_action", args)
|
||||
return _result(
|
||||
"control_action",
|
||||
receipt,
|
||||
preflight=inspection,
|
||||
node_before=receipt.get("node_before", node),
|
||||
node_after=receipt.get("node_after"),
|
||||
verification="pending_observation",
|
||||
)
|
||||
|
||||
|
||||
async def _scroll(request: ScrollToRequest, native: NativeCaller) -> dict:
|
||||
started = monotonic()
|
||||
deadline = started + request.timeout_ms / 1000
|
||||
container_selector = request.container.model_dump(mode="json", exclude_none=True)
|
||||
target_args = {
|
||||
"selector": request.target.model_dump(mode="json", exclude_none=True),
|
||||
"scope": container_selector,
|
||||
}
|
||||
observations: list[dict] = []
|
||||
scroll_results: list[dict] = []
|
||||
last_container: dict = {}
|
||||
last_target: dict = {}
|
||||
|
||||
def finish(receipt: dict) -> dict:
|
||||
result = dict(receipt)
|
||||
if result.get("execution") not in {"unknown", "cancelled"} and any(
|
||||
item["native_receipt"]["success"] for item in scroll_results
|
||||
):
|
||||
result["execution"] = "completed"
|
||||
return _result(
|
||||
"scroll_to",
|
||||
result,
|
||||
node=last_target.get("node"),
|
||||
container=last_container.get("node"),
|
||||
scroll_results=scroll_results,
|
||||
observations=observations,
|
||||
verification="pending_observation",
|
||||
)
|
||||
|
||||
async def inspect(args: dict) -> dict:
|
||||
receipt = await _inspect(native, args, timeout=max(0, deadline - monotonic()))
|
||||
_remember(observations, started, receipt)
|
||||
if monotonic() > deadline:
|
||||
return _failure("TIMEOUT", "The native observation arrived after the search deadline.")
|
||||
return receipt
|
||||
|
||||
def target_visible(receipt: dict) -> bool:
|
||||
if receipt["success"]:
|
||||
return receipt["node"].get("visible") is True
|
||||
return False
|
||||
|
||||
def target_state(receipt: dict) -> tuple:
|
||||
# A target may be outside the bounded container signature sample.
|
||||
# Its own visibility and identity must independently remain quiet too.
|
||||
return (True, receipt["node"].get("token")) if target_visible(receipt) else (False,)
|
||||
|
||||
def target_error(receipt: dict) -> dict | None:
|
||||
if not receipt["success"] and receipt.get("code") != "NOT_FOUND":
|
||||
return receipt
|
||||
if receipt["success"] and type(receipt["node"].get("visible")) is not bool:
|
||||
return _failure("INVALID_RESPONSE", "The scoped target has no native visibility state.")
|
||||
return None
|
||||
|
||||
for _ in range(request.max_scrolls):
|
||||
if monotonic() >= deadline:
|
||||
break
|
||||
last_container = await inspect({"selector": container_selector})
|
||||
if not last_container["success"]:
|
||||
return finish(last_container)
|
||||
last_target = await inspect(target_args)
|
||||
error = target_error(last_target)
|
||||
if error is not None:
|
||||
return finish(error)
|
||||
if target_visible(last_target):
|
||||
return finish({"success": True, "execution": "observed", "code": "OK"})
|
||||
node = last_container["node"]
|
||||
action = f"scroll_{request.direction}"
|
||||
invalid = _write_guard(node, action)
|
||||
signature = node.get("content_signature")
|
||||
if invalid is not None:
|
||||
return finish(invalid)
|
||||
if not isinstance(signature, str) or not signature:
|
||||
return finish(_failure("INVALID_RESPONSE", "The container has no content_signature."))
|
||||
if monotonic() >= deadline:
|
||||
break
|
||||
receipt = await _call(
|
||||
native,
|
||||
"control_action",
|
||||
{"selector": container_selector, "action": action, "expected_token": node["token"]},
|
||||
timeout=CONTROL_COMPLETION_SECONDS,
|
||||
)
|
||||
scroll_results.append({"node_before": node, "native_receipt": receipt})
|
||||
if not receipt["success"]:
|
||||
if receipt.get("code") == "ACTION_REJECTED" and receipt["execution"] == "not_started":
|
||||
return finish(
|
||||
_failure("NO_PROGRESS", "The native container rejected further scrolling.")
|
||||
)
|
||||
return finish(receipt)
|
||||
if monotonic() >= deadline:
|
||||
break
|
||||
|
||||
settle_seconds = request.settle_ms / 1000
|
||||
# A native scroll accepts the animation; it does not wait for layout.
|
||||
# Require a quiet content interval before the next scroll, including when
|
||||
# the target briefly enters the viewport during that animation. Also give
|
||||
# a delayed first change a short grace rather than reissuing input.
|
||||
no_progress_deadline = min(deadline, monotonic() + max(settle_seconds, 0.5))
|
||||
stable_since = monotonic()
|
||||
last_signature = (signature, target_state(last_target))
|
||||
changed = False
|
||||
while True:
|
||||
if monotonic() >= deadline:
|
||||
break
|
||||
last_target = await inspect(target_args)
|
||||
error = target_error(last_target)
|
||||
if error is not None:
|
||||
return finish(error)
|
||||
last_container = await inspect({"selector": container_selector})
|
||||
if not last_container["success"]:
|
||||
return finish(last_container)
|
||||
current = last_container["node"].get("content_signature")
|
||||
if not isinstance(current, str) or not current:
|
||||
return finish(
|
||||
_failure("INVALID_RESPONSE", "The container has no content_signature.")
|
||||
)
|
||||
now = monotonic()
|
||||
current_state = (current, target_state(last_target))
|
||||
if current_state != last_signature:
|
||||
changed = True
|
||||
stable_since = now
|
||||
last_signature = current_state
|
||||
if changed and now - stable_since >= settle_seconds - 1e-9:
|
||||
break
|
||||
if not changed and now >= no_progress_deadline - 1e-9:
|
||||
break
|
||||
await _pause(deadline, 0.05)
|
||||
if monotonic() >= deadline:
|
||||
break
|
||||
if target_visible(last_target):
|
||||
return finish({"success": True, "execution": "completed", "code": "OK"})
|
||||
if not changed:
|
||||
return finish(
|
||||
_failure("NO_PROGRESS", "The container content did not change after scrolling.")
|
||||
)
|
||||
code = "TIMEOUT" if monotonic() >= deadline else "MAX_SCROLLS"
|
||||
return finish(
|
||||
_failure(code, "The visible target was not found within the bounded scroll search.")
|
||||
)
|
||||
|
||||
|
||||
async def execute_native(
|
||||
name: str,
|
||||
request: BaseModel | dict,
|
||||
width: int,
|
||||
height: int,
|
||||
call_native: NativeCaller,
|
||||
) -> dict:
|
||||
"""Execute one validated generic action without retrying any write.
|
||||
|
||||
Cancellation always propagates. Structured helper receipts remain available
|
||||
even when a surrounding wait/scroll condition was not established.
|
||||
"""
|
||||
try:
|
||||
model = NATIVE_REQUESTS.get(name)
|
||||
if model is None:
|
||||
raise ValueError("Unknown native action.")
|
||||
args = request.model_dump(mode="json") if isinstance(request, BaseModel) else request
|
||||
parsed = model.model_validate(args)
|
||||
if isinstance(parsed, (PinchRequest, TouchGestureRequest)):
|
||||
planned = (
|
||||
plan_pinch(parsed, width, height)
|
||||
if isinstance(parsed, PinchRequest)
|
||||
else plan_touch(parsed, width, height)
|
||||
)
|
||||
else:
|
||||
planned = None
|
||||
except (ValidationError, TypeError, ValueError) as exc:
|
||||
return _result(name, _failure("INVALID_ARGUMENT", str(exc)))
|
||||
|
||||
if planned is not None:
|
||||
operation_id = str(uuid4())
|
||||
args = {
|
||||
"operation_id": operation_id,
|
||||
"frames": planned["frames"],
|
||||
"screen_size": planned["screen_size"],
|
||||
}
|
||||
if planned.get("require_visible"):
|
||||
args["require_visible"] = planned["require_visible"]
|
||||
receipt = await _call(
|
||||
call_native, "touch_gesture", args, timeout=planned["frames"][-1]["at_ms"] / 1000 + 5
|
||||
)
|
||||
return _result(
|
||||
name,
|
||||
receipt,
|
||||
operation_id=operation_id,
|
||||
plan=planned,
|
||||
verification="pending_observation",
|
||||
)
|
||||
if isinstance(parsed, WaitUntilRequest):
|
||||
return await _wait(parsed, call_native)
|
||||
if isinstance(parsed, ControlActionRequest):
|
||||
return await _control(parsed, call_native)
|
||||
if isinstance(parsed, ScrollToRequest):
|
||||
return await _scroll(parsed, call_native)
|
||||
if isinstance(parsed, ReadControlRequest):
|
||||
receipt = await _inspect(call_native, {"selector": _selector(parsed)})
|
||||
return _result(name, receipt)
|
||||
return _result(
|
||||
name, _failure("INVALID_ARGUMENT", "The action has no native execution handler.")
|
||||
)
|
||||
Reference in new issue
Block a user