feat(android): add native controls and continuous multi-touch
This commit is contained in:
chenmingxuan committed
1 parent
3dd96078db
commit
4b12ae7728
70 files changed
+13105
-179
No files matched your search
@@ -0,0 +1,499 @@
|
||||
"""Bounded native operations; accepted input is not a business verification."""
|
||||
|
||||
import asyncio
|
||||
from collections.abc import Awaitable
|
||||
import math
|
||||
from time import monotonic
|
||||
from typing import Any, Protocol
|
||||
from uuid import uuid4
|
||||
|
||||
from pydantic import BaseModel, ValidationError
|
||||
|
||||
from artemis.mcp.native_actions import (
|
||||
NATIVE_REQUESTS,
|
||||
ControlActionRequest,
|
||||
PinchRequest,
|
||||
ReadControlRequest,
|
||||
ScrollToRequest,
|
||||
TouchGestureRequest,
|
||||
WaitUntilRequest,
|
||||
plan_pinch,
|
||||
plan_touch,
|
||||
)
|
||||
from artemis.runtime.native_protocol import CONTROL_COMPLETION_SECONDS
|
||||
|
||||
sleep = asyncio.sleep
|
||||
_EXECUTIONS = {"observed", "completed", "not_started", "unknown", "cancelled"}
|
||||
_OBSERVATION_LIMIT = 64
|
||||
|
||||
|
||||
class NativeCaller(Protocol):
|
||||
def __call__(
|
||||
self, command: str, args: dict, *, timeout: float | None = None
|
||||
) -> Awaitable[dict]: ...
|
||||
|
||||
|
||||
def _failure(code: str, error: str, *, execution: str = "not_started", **evidence: Any) -> dict:
|
||||
return {"success": False, "execution": execution, "code": code, "error": error, **evidence}
|
||||
|
||||
|
||||
def _result(name: str, receipt: dict, **evidence: Any) -> dict:
|
||||
result = {**receipt, **evidence, "action": name}
|
||||
result.setdefault("native_receipt", dict(receipt))
|
||||
return result
|
||||
|
||||
|
||||
async def _call(
|
||||
native: NativeCaller, command: str, args: dict, *, timeout: float | None = None
|
||||
) -> dict:
|
||||
mutation = command != "inspect_control"
|
||||
if timeout is not None and timeout <= 0:
|
||||
return _failure("TIMEOUT", "The operation deadline expired before native dispatch.")
|
||||
try:
|
||||
if timeout is None:
|
||||
result = await native(command, args, timeout=None)
|
||||
else:
|
||||
async with asyncio.timeout(timeout):
|
||||
result = await native(command, args, timeout=timeout)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except (OSError, RuntimeError, ValueError) as exc:
|
||||
return _failure(
|
||||
"TIMEOUT" if isinstance(exc, TimeoutError) else "NATIVE_ERROR",
|
||||
f"Native {command} failed: {exc}",
|
||||
execution="unknown" if mutation else "not_started",
|
||||
)
|
||||
if (
|
||||
not isinstance(result, dict)
|
||||
or type(result.get("success")) is not bool
|
||||
or not isinstance(result.get("execution"), str)
|
||||
or result.get("execution") not in _EXECUTIONS
|
||||
or not isinstance(result.get("code"), str)
|
||||
or not result.get("code")
|
||||
):
|
||||
return _failure(
|
||||
"INVALID_RESPONSE",
|
||||
"The native operation did not return a structured execution receipt.",
|
||||
execution="unknown" if mutation else "not_started",
|
||||
native_receipt=result,
|
||||
)
|
||||
if result["success"] and result["execution"] != ("completed" if mutation else "observed"):
|
||||
return _failure(
|
||||
result["code"] if result["code"] != "OK" else "INVALID_RESPONSE",
|
||||
"The native receipt does not establish completion of this operation.",
|
||||
execution=result["execution"],
|
||||
native_receipt=result,
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
async def _inspect(native: NativeCaller, args: dict, *, timeout: float | None = None) -> dict:
|
||||
receipt = await _call(native, "inspect_control", args, timeout=timeout)
|
||||
if receipt["success"]:
|
||||
if receipt.get("candidate_count", 1) != 1:
|
||||
return _failure(
|
||||
"AMBIGUOUS", "The selector did not resolve uniquely.", native_receipt=receipt
|
||||
)
|
||||
if not isinstance(receipt.get("node"), dict):
|
||||
return _failure(
|
||||
"INVALID_RESPONSE", "The native read has no node.", native_receipt=receipt
|
||||
)
|
||||
return receipt
|
||||
|
||||
|
||||
def _selector(request: ReadControlRequest) -> dict:
|
||||
return request.selector.model_dump(mode="json", exclude_none=True)
|
||||
|
||||
|
||||
def _identity(node: dict) -> tuple[str, str] | None:
|
||||
token, content = node.get("token"), node.get("content_signature")
|
||||
if isinstance(token, str) and token and isinstance(content, str) and content:
|
||||
return token, content
|
||||
return None
|
||||
|
||||
|
||||
def _range(node: dict) -> dict | None:
|
||||
value = node.get("range")
|
||||
if not isinstance(value, dict):
|
||||
return None
|
||||
values = [value.get(key) for key in ("min", "max", "current")]
|
||||
if not all(
|
||||
type(number) in (int, float) and -1e308 <= number <= 1e308 and math.isfinite(number)
|
||||
for number in values
|
||||
):
|
||||
return None
|
||||
if not value["min"] <= value["current"] <= value["max"]:
|
||||
return None
|
||||
return value
|
||||
|
||||
|
||||
def _write_guard(node: dict, action: str, expected_token: str | None = None) -> dict | None:
|
||||
token = node.get("token")
|
||||
if not isinstance(token, str) or not token:
|
||||
return _failure("INVALID_RESPONSE", "The node has no fresh identity token.")
|
||||
if expected_token is not None and token != expected_token:
|
||||
return _failure("STALE", "The node changed since the supplied observation.")
|
||||
actions = node.get("actions")
|
||||
if not isinstance(actions, list) or action not in actions:
|
||||
return _failure(
|
||||
"UNSUPPORTED", "The node does not advertise the requested canonical action."
|
||||
)
|
||||
if node.get("enabled") is not True:
|
||||
return _failure("DISABLED", "The native node is not confirmed enabled.")
|
||||
return None
|
||||
|
||||
|
||||
def _remember(observations: list[dict], started: float, receipt: dict) -> None:
|
||||
# Keep the first observation and the most recent bounded evidence window.
|
||||
if len(observations) == _OBSERVATION_LIMIT:
|
||||
del observations[1]
|
||||
observations.append({"at_ms": round((monotonic() - started) * 1000), **receipt})
|
||||
|
||||
|
||||
async def _pause(deadline: float, interval: float) -> None:
|
||||
remaining = deadline - monotonic()
|
||||
if remaining > 0:
|
||||
await sleep(min(interval, remaining))
|
||||
|
||||
|
||||
def _condition(request: WaitUntilRequest, node: dict) -> bool:
|
||||
state = request.state
|
||||
if state == "text":
|
||||
if node.get("password") is True or not isinstance(node.get("text"), str):
|
||||
raise ValueError("An exact readable native text value is required.")
|
||||
return node["text"] == request.expected_text
|
||||
if state == "value":
|
||||
native_range = _range(node)
|
||||
if native_range is None or request.expected_value is None:
|
||||
raise ValueError("A finite native min/max/current range is required.")
|
||||
return math.isclose(
|
||||
native_range["current"], request.expected_value, rel_tol=0, abs_tol=request.tolerance
|
||||
)
|
||||
field, expected = {
|
||||
"visible": ("visible", True),
|
||||
"absent": ("visible", False),
|
||||
"enabled": ("enabled", True),
|
||||
"disabled": ("enabled", False),
|
||||
"checked": ("checked", True),
|
||||
"unchecked": ("checked", False),
|
||||
"selected": ("selected", True),
|
||||
"unselected": ("selected", False),
|
||||
}[state]
|
||||
if state in {"checked", "unchecked"} and node.get("checkable") is not True:
|
||||
raise ValueError("A checked predicate requires an explicitly checkable node.")
|
||||
if type(node.get(field)) is not bool:
|
||||
raise ValueError(f"The native node does not expose a Boolean {field} state.")
|
||||
return node[field] is expected
|
||||
|
||||
|
||||
async def _wait(request: WaitUntilRequest, native: NativeCaller) -> dict:
|
||||
started = monotonic()
|
||||
deadline = started + request.timeout_ms / 1000
|
||||
observations: list[dict] = []
|
||||
attempts = 0
|
||||
stable_key = None
|
||||
stable_since = started
|
||||
stable_samples = 0
|
||||
last: dict = {}
|
||||
while monotonic() < deadline:
|
||||
remaining = deadline - monotonic()
|
||||
if remaining <= 0:
|
||||
break
|
||||
last = await _inspect(native, {"selector": _selector(request)}, timeout=remaining)
|
||||
attempts += 1
|
||||
_remember(observations, started, last)
|
||||
if monotonic() > deadline:
|
||||
break
|
||||
matched = False
|
||||
if not last["success"]:
|
||||
code = last.get("code")
|
||||
if code not in {"NOT_FOUND", "READ_FAILED", "UNAVAILABLE"}:
|
||||
return _result("wait_until", last, observations=observations, attempts=attempts)
|
||||
# Window teardown can temporarily make a declared root unreadable.
|
||||
# Poll only reads within the original deadline; an incomplete read
|
||||
# proves neither absence nor continuity of a previous stable sample.
|
||||
matched = code == "NOT_FOUND" and request.state == "absent"
|
||||
stable_key, stable_samples = None, 0
|
||||
elif request.state == "stable":
|
||||
key = _identity(last["node"])
|
||||
if key is None:
|
||||
return _result(
|
||||
"wait_until",
|
||||
_failure(
|
||||
"INVALID_RESPONSE", "Stability needs both token and content_signature."
|
||||
),
|
||||
node=last["node"],
|
||||
observations=observations,
|
||||
attempts=attempts,
|
||||
)
|
||||
now = monotonic()
|
||||
if key != stable_key:
|
||||
stable_key, stable_since, stable_samples = key, now, 1
|
||||
else:
|
||||
stable_samples += 1
|
||||
matched = stable_samples >= 2 and (now - stable_since) * 1000 >= request.stable_ms
|
||||
else:
|
||||
try:
|
||||
matched = _condition(request, last["node"])
|
||||
except ValueError as exc:
|
||||
return _result(
|
||||
"wait_until",
|
||||
_failure("INVALID_STATE", str(exc)),
|
||||
node=last["node"],
|
||||
observations=observations,
|
||||
attempts=attempts,
|
||||
)
|
||||
if matched:
|
||||
return _result(
|
||||
"wait_until",
|
||||
{"success": True, "execution": "observed", "code": "OK"},
|
||||
native_receipt=last,
|
||||
state=request.state,
|
||||
node=last.get("node"),
|
||||
observations=observations,
|
||||
attempts=attempts,
|
||||
)
|
||||
await _pause(deadline, request.poll_interval_ms / 1000)
|
||||
return _result(
|
||||
"wait_until",
|
||||
_failure("TIMEOUT", "The requested native state was not observed within the deadline."),
|
||||
native_receipt=last,
|
||||
observations=observations,
|
||||
attempts=attempts,
|
||||
)
|
||||
|
||||
|
||||
async def _control(request: ControlActionRequest, native: NativeCaller) -> dict:
|
||||
inspection = await _inspect(native, {"selector": _selector(request)})
|
||||
if not inspection["success"]:
|
||||
return _result("control_action", inspection)
|
||||
node = inspection["node"]
|
||||
invalid = _write_guard(node, request.action, request.expected_token)
|
||||
if invalid is None and request.action == "set_progress":
|
||||
native_range = _range(node)
|
||||
value = request.value
|
||||
if (
|
||||
native_range is None
|
||||
or value is None
|
||||
or not math.isfinite(value)
|
||||
or not native_range["min"] <= value <= native_range["max"]
|
||||
or (native_range.get("type") == 0 and value != int(value))
|
||||
):
|
||||
invalid = _failure(
|
||||
"INVALID_ARGUMENT", "The value is incompatible with the live native range."
|
||||
)
|
||||
if invalid is not None:
|
||||
return _result("control_action", invalid, preflight=inspection, node_before=node)
|
||||
args = request.model_dump(mode="json", exclude_none=True)
|
||||
args["expected_token"] = node["token"]
|
||||
receipt = await _call(native, "control_action", args)
|
||||
return _result(
|
||||
"control_action",
|
||||
receipt,
|
||||
preflight=inspection,
|
||||
node_before=receipt.get("node_before", node),
|
||||
node_after=receipt.get("node_after"),
|
||||
verification="pending_observation",
|
||||
)
|
||||
|
||||
|
||||
async def _scroll(request: ScrollToRequest, native: NativeCaller) -> dict:
|
||||
started = monotonic()
|
||||
deadline = started + request.timeout_ms / 1000
|
||||
container_selector = request.container.model_dump(mode="json", exclude_none=True)
|
||||
target_args = {
|
||||
"selector": request.target.model_dump(mode="json", exclude_none=True),
|
||||
"scope": container_selector,
|
||||
}
|
||||
observations: list[dict] = []
|
||||
scroll_results: list[dict] = []
|
||||
last_container: dict = {}
|
||||
last_target: dict = {}
|
||||
|
||||
def finish(receipt: dict) -> dict:
|
||||
result = dict(receipt)
|
||||
if result.get("execution") not in {"unknown", "cancelled"} and any(
|
||||
item["native_receipt"]["success"] for item in scroll_results
|
||||
):
|
||||
result["execution"] = "completed"
|
||||
return _result(
|
||||
"scroll_to",
|
||||
result,
|
||||
node=last_target.get("node"),
|
||||
container=last_container.get("node"),
|
||||
scroll_results=scroll_results,
|
||||
observations=observations,
|
||||
verification="pending_observation",
|
||||
)
|
||||
|
||||
async def inspect(args: dict) -> dict:
|
||||
receipt = await _inspect(native, args, timeout=max(0, deadline - monotonic()))
|
||||
_remember(observations, started, receipt)
|
||||
if monotonic() > deadline:
|
||||
return _failure("TIMEOUT", "The native observation arrived after the search deadline.")
|
||||
return receipt
|
||||
|
||||
def target_visible(receipt: dict) -> bool:
|
||||
if receipt["success"]:
|
||||
return receipt["node"].get("visible") is True
|
||||
return False
|
||||
|
||||
def target_state(receipt: dict) -> tuple:
|
||||
# A target may be outside the bounded container signature sample.
|
||||
# Its own visibility and identity must independently remain quiet too.
|
||||
return (True, receipt["node"].get("token")) if target_visible(receipt) else (False,)
|
||||
|
||||
def target_error(receipt: dict) -> dict | None:
|
||||
if not receipt["success"] and receipt.get("code") != "NOT_FOUND":
|
||||
return receipt
|
||||
if receipt["success"] and type(receipt["node"].get("visible")) is not bool:
|
||||
return _failure("INVALID_RESPONSE", "The scoped target has no native visibility state.")
|
||||
return None
|
||||
|
||||
for _ in range(request.max_scrolls):
|
||||
if monotonic() >= deadline:
|
||||
break
|
||||
last_container = await inspect({"selector": container_selector})
|
||||
if not last_container["success"]:
|
||||
return finish(last_container)
|
||||
last_target = await inspect(target_args)
|
||||
error = target_error(last_target)
|
||||
if error is not None:
|
||||
return finish(error)
|
||||
if target_visible(last_target):
|
||||
return finish({"success": True, "execution": "observed", "code": "OK"})
|
||||
node = last_container["node"]
|
||||
action = f"scroll_{request.direction}"
|
||||
invalid = _write_guard(node, action)
|
||||
signature = node.get("content_signature")
|
||||
if invalid is not None:
|
||||
return finish(invalid)
|
||||
if not isinstance(signature, str) or not signature:
|
||||
return finish(_failure("INVALID_RESPONSE", "The container has no content_signature."))
|
||||
if monotonic() >= deadline:
|
||||
break
|
||||
receipt = await _call(
|
||||
native,
|
||||
"control_action",
|
||||
{"selector": container_selector, "action": action, "expected_token": node["token"]},
|
||||
timeout=CONTROL_COMPLETION_SECONDS,
|
||||
)
|
||||
scroll_results.append({"node_before": node, "native_receipt": receipt})
|
||||
if not receipt["success"]:
|
||||
if receipt.get("code") == "ACTION_REJECTED" and receipt["execution"] == "not_started":
|
||||
return finish(
|
||||
_failure("NO_PROGRESS", "The native container rejected further scrolling.")
|
||||
)
|
||||
return finish(receipt)
|
||||
if monotonic() >= deadline:
|
||||
break
|
||||
|
||||
settle_seconds = request.settle_ms / 1000
|
||||
# A native scroll accepts the animation; it does not wait for layout.
|
||||
# Require a quiet content interval before the next scroll, including when
|
||||
# the target briefly enters the viewport during that animation. Also give
|
||||
# a delayed first change a short grace rather than reissuing input.
|
||||
no_progress_deadline = min(deadline, monotonic() + max(settle_seconds, 0.5))
|
||||
stable_since = monotonic()
|
||||
last_signature = (signature, target_state(last_target))
|
||||
changed = False
|
||||
while True:
|
||||
if monotonic() >= deadline:
|
||||
break
|
||||
last_target = await inspect(target_args)
|
||||
error = target_error(last_target)
|
||||
if error is not None:
|
||||
return finish(error)
|
||||
last_container = await inspect({"selector": container_selector})
|
||||
if not last_container["success"]:
|
||||
return finish(last_container)
|
||||
current = last_container["node"].get("content_signature")
|
||||
if not isinstance(current, str) or not current:
|
||||
return finish(
|
||||
_failure("INVALID_RESPONSE", "The container has no content_signature.")
|
||||
)
|
||||
now = monotonic()
|
||||
current_state = (current, target_state(last_target))
|
||||
if current_state != last_signature:
|
||||
changed = True
|
||||
stable_since = now
|
||||
last_signature = current_state
|
||||
if changed and now - stable_since >= settle_seconds - 1e-9:
|
||||
break
|
||||
if not changed and now >= no_progress_deadline - 1e-9:
|
||||
break
|
||||
await _pause(deadline, 0.05)
|
||||
if monotonic() >= deadline:
|
||||
break
|
||||
if target_visible(last_target):
|
||||
return finish({"success": True, "execution": "completed", "code": "OK"})
|
||||
if not changed:
|
||||
return finish(
|
||||
_failure("NO_PROGRESS", "The container content did not change after scrolling.")
|
||||
)
|
||||
code = "TIMEOUT" if monotonic() >= deadline else "MAX_SCROLLS"
|
||||
return finish(
|
||||
_failure(code, "The visible target was not found within the bounded scroll search.")
|
||||
)
|
||||
|
||||
|
||||
async def execute_native(
|
||||
name: str,
|
||||
request: BaseModel | dict,
|
||||
width: int,
|
||||
height: int,
|
||||
call_native: NativeCaller,
|
||||
) -> dict:
|
||||
"""Execute one validated generic action without retrying any write.
|
||||
|
||||
Cancellation always propagates. Structured helper receipts remain available
|
||||
even when a surrounding wait/scroll condition was not established.
|
||||
"""
|
||||
try:
|
||||
model = NATIVE_REQUESTS.get(name)
|
||||
if model is None:
|
||||
raise ValueError("Unknown native action.")
|
||||
args = request.model_dump(mode="json") if isinstance(request, BaseModel) else request
|
||||
parsed = model.model_validate(args)
|
||||
if isinstance(parsed, (PinchRequest, TouchGestureRequest)):
|
||||
planned = (
|
||||
plan_pinch(parsed, width, height)
|
||||
if isinstance(parsed, PinchRequest)
|
||||
else plan_touch(parsed, width, height)
|
||||
)
|
||||
else:
|
||||
planned = None
|
||||
except (ValidationError, TypeError, ValueError) as exc:
|
||||
return _result(name, _failure("INVALID_ARGUMENT", str(exc)))
|
||||
|
||||
if planned is not None:
|
||||
operation_id = str(uuid4())
|
||||
args = {
|
||||
"operation_id": operation_id,
|
||||
"frames": planned["frames"],
|
||||
"screen_size": planned["screen_size"],
|
||||
}
|
||||
if planned.get("require_visible"):
|
||||
args["require_visible"] = planned["require_visible"]
|
||||
receipt = await _call(
|
||||
call_native, "touch_gesture", args, timeout=planned["frames"][-1]["at_ms"] / 1000 + 5
|
||||
)
|
||||
return _result(
|
||||
name,
|
||||
receipt,
|
||||
operation_id=operation_id,
|
||||
plan=planned,
|
||||
verification="pending_observation",
|
||||
)
|
||||
if isinstance(parsed, WaitUntilRequest):
|
||||
return await _wait(parsed, call_native)
|
||||
if isinstance(parsed, ControlActionRequest):
|
||||
return await _control(parsed, call_native)
|
||||
if isinstance(parsed, ScrollToRequest):
|
||||
return await _scroll(parsed, call_native)
|
||||
if isinstance(parsed, ReadControlRequest):
|
||||
receipt = await _inspect(call_native, {"selector": _selector(parsed)})
|
||||
return _result(name, receipt)
|
||||
return _result(
|
||||
name, _failure("INVALID_ARGUMENT", "The action has no native execution handler.")
|
||||
)
|
||||
Reference in new issue
Block a user