chenmingxuan
4ea7939d9c
Add optional ensure_state, gesture region and final value feedback shared by Flash and Pro. Preserve completed inputs when verification fails and stop repeated ineffective strategies. Regenerate pinned tool schemas for the new optional arguments.
538 lines
21 KiB
Python
538 lines
21 KiB
Python
"""Bounded native operations; accepted input is not a business verification."""
|
|
|
|
import asyncio
|
|
from collections.abc import Awaitable
|
|
import math
|
|
from time import monotonic
|
|
from typing import Any, Protocol
|
|
from uuid import uuid4
|
|
|
|
from pydantic import BaseModel, ValidationError
|
|
|
|
from artemis.mcp.native_actions import (
|
|
NATIVE_REQUESTS,
|
|
ControlActionRequest,
|
|
PinchRequest,
|
|
ReadControlRequest,
|
|
ScrollToRequest,
|
|
TouchGestureRequest,
|
|
WaitUntilRequest,
|
|
plan_pinch,
|
|
plan_touch,
|
|
)
|
|
from artemis.runtime.native_protocol import CONTROL_COMPLETION_SECONDS
|
|
|
|
sleep = asyncio.sleep
|
|
_EXECUTIONS = {"observed", "completed", "not_started", "unknown", "cancelled"}
|
|
_OBSERVATION_LIMIT = 64
|
|
|
|
|
|
class NativeCaller(Protocol):
|
|
def __call__(
|
|
self, command: str, args: dict, *, timeout: float | None = None
|
|
) -> Awaitable[dict]: ...
|
|
|
|
|
|
def _failure(code: str, error: str, *, execution: str = "not_started", **evidence: Any) -> dict:
|
|
return {"success": False, "execution": execution, "code": code, "error": error, **evidence}
|
|
|
|
|
|
def _result(name: str, receipt: dict, **evidence: Any) -> dict:
|
|
result = {**receipt, **evidence, "action": name}
|
|
result.setdefault("native_receipt", dict(receipt))
|
|
return result
|
|
|
|
|
|
async def _call(
|
|
native: NativeCaller, command: str, args: dict, *, timeout: float | None = None
|
|
) -> dict:
|
|
mutation = command != "inspect_control"
|
|
if timeout is not None and timeout <= 0:
|
|
return _failure("TIMEOUT", "The operation deadline expired before native dispatch.")
|
|
try:
|
|
if timeout is None:
|
|
result = await native(command, args, timeout=None)
|
|
else:
|
|
async with asyncio.timeout(timeout):
|
|
result = await native(command, args, timeout=timeout)
|
|
except asyncio.CancelledError:
|
|
raise
|
|
except (OSError, RuntimeError, ValueError) as exc:
|
|
return _failure(
|
|
"TIMEOUT" if isinstance(exc, TimeoutError) else "NATIVE_ERROR",
|
|
f"Native {command} failed: {exc}",
|
|
execution="unknown" if mutation else "not_started",
|
|
)
|
|
if (
|
|
not isinstance(result, dict)
|
|
or type(result.get("success")) is not bool
|
|
or not isinstance(result.get("execution"), str)
|
|
or result.get("execution") not in _EXECUTIONS
|
|
or not isinstance(result.get("code"), str)
|
|
or not result.get("code")
|
|
):
|
|
return _failure(
|
|
"INVALID_RESPONSE",
|
|
"The native operation did not return a structured execution receipt.",
|
|
execution="unknown" if mutation else "not_started",
|
|
native_receipt=result,
|
|
)
|
|
if result["success"] and result["execution"] != ("completed" if mutation else "observed"):
|
|
return _failure(
|
|
result["code"] if result["code"] != "OK" else "INVALID_RESPONSE",
|
|
"The native receipt does not establish completion of this operation.",
|
|
execution=result["execution"],
|
|
native_receipt=result,
|
|
)
|
|
return result
|
|
|
|
|
|
async def _inspect(native: NativeCaller, args: dict, *, timeout: float | None = None) -> dict:
|
|
receipt = await _call(native, "inspect_control", args, timeout=timeout)
|
|
if receipt["success"]:
|
|
if receipt.get("candidate_count", 1) != 1:
|
|
return _failure(
|
|
"AMBIGUOUS", "The selector did not resolve uniquely.", native_receipt=receipt
|
|
)
|
|
if not isinstance(receipt.get("node"), dict):
|
|
return _failure(
|
|
"INVALID_RESPONSE", "The native read has no node.", native_receipt=receipt
|
|
)
|
|
return receipt
|
|
|
|
|
|
def _selector(request: ReadControlRequest) -> dict:
|
|
return request.selector.model_dump(mode="json", exclude_none=True)
|
|
|
|
|
|
def _identity(node: dict) -> tuple[str, str] | None:
|
|
token, content = node.get("token"), node.get("content_signature")
|
|
if isinstance(token, str) and token and isinstance(content, str) and content:
|
|
return token, content
|
|
return None
|
|
|
|
|
|
def _range(node: dict) -> dict | None:
|
|
value = node.get("range")
|
|
if not isinstance(value, dict):
|
|
return None
|
|
values = [value.get(key) for key in ("min", "max", "current")]
|
|
if not all(
|
|
type(number) in (int, float) and -1e308 <= number <= 1e308 and math.isfinite(number)
|
|
for number in values
|
|
):
|
|
return None
|
|
if not value["min"] <= value["current"] <= value["max"]:
|
|
return None
|
|
return value
|
|
|
|
|
|
def _write_guard(node: dict, action: str, expected_token: str | None = None) -> dict | None:
|
|
token = node.get("token")
|
|
if not isinstance(token, str) or not token:
|
|
return _failure("INVALID_RESPONSE", "The node has no fresh identity token.")
|
|
if expected_token is not None and token != expected_token:
|
|
return _failure("STALE", "The node changed since the supplied observation.")
|
|
actions = node.get("actions")
|
|
if not isinstance(actions, list) or action not in actions:
|
|
return _failure(
|
|
"UNSUPPORTED", "The node does not advertise the requested canonical action."
|
|
)
|
|
if node.get("enabled") is not True:
|
|
return _failure("DISABLED", "The native node is not confirmed enabled.")
|
|
return None
|
|
|
|
|
|
def _remember(observations: list[dict], started: float, receipt: dict) -> None:
|
|
# Keep the first observation and the most recent bounded evidence window.
|
|
if len(observations) == _OBSERVATION_LIMIT:
|
|
del observations[1]
|
|
observations.append({"at_ms": round((monotonic() - started) * 1000), **receipt})
|
|
|
|
|
|
async def _pause(deadline: float, interval: float) -> None:
|
|
remaining = deadline - monotonic()
|
|
if remaining > 0:
|
|
await sleep(min(interval, remaining))
|
|
|
|
|
|
def _condition(request: WaitUntilRequest, node: dict) -> bool:
|
|
state = request.state
|
|
if state in {"text", "content_desc"}:
|
|
if node.get("password") is True or not isinstance(node.get(state), str):
|
|
raise ValueError("An exact readable native text value is required.")
|
|
return node[state] == request.expected_text
|
|
if state == "value":
|
|
native_range = _range(node)
|
|
if native_range is None or request.expected_value is None:
|
|
raise ValueError("A finite native min/max/current range is required.")
|
|
return math.isclose(
|
|
native_range["current"], request.expected_value, rel_tol=0, abs_tol=request.tolerance
|
|
)
|
|
field, expected = {
|
|
"visible": ("visible", True),
|
|
"absent": ("visible", False),
|
|
"enabled": ("enabled", True),
|
|
"disabled": ("enabled", False),
|
|
"checked": ("checked", True),
|
|
"unchecked": ("checked", False),
|
|
"selected": ("selected", True),
|
|
"unselected": ("selected", False),
|
|
}[state]
|
|
if state in {"checked", "unchecked"} and node.get("checkable") is not True:
|
|
raise ValueError("A checked predicate requires an explicitly checkable node.")
|
|
if type(node.get(field)) is not bool:
|
|
raise ValueError(f"The native node does not expose a Boolean {field} state.")
|
|
return node[field] is expected
|
|
|
|
|
|
async def _wait(request: WaitUntilRequest, native: NativeCaller) -> dict:
|
|
started = monotonic()
|
|
deadline = started + request.timeout_ms / 1000
|
|
observations: list[dict] = []
|
|
attempts = 0
|
|
stable_key = None
|
|
stable_since = started
|
|
stable_samples = 0
|
|
last: dict = {}
|
|
while monotonic() < deadline:
|
|
remaining = deadline - monotonic()
|
|
if remaining <= 0:
|
|
break
|
|
last = await _inspect(native, {"selector": _selector(request)}, timeout=remaining)
|
|
attempts += 1
|
|
_remember(observations, started, last)
|
|
if monotonic() > deadline:
|
|
break
|
|
matched = False
|
|
if not last["success"]:
|
|
code = last.get("code")
|
|
if code not in {"NOT_FOUND", "READ_FAILED", "UNAVAILABLE"}:
|
|
return _result("wait_until", last, observations=observations, attempts=attempts)
|
|
# Window teardown can temporarily make a declared root unreadable.
|
|
# Poll only reads within the original deadline; an incomplete read
|
|
# proves neither absence nor continuity of a previous stable sample.
|
|
matched = code == "NOT_FOUND" and request.state == "absent"
|
|
stable_key, stable_samples = None, 0
|
|
elif request.state == "stable":
|
|
key = _identity(last["node"])
|
|
if key is None:
|
|
return _result(
|
|
"wait_until",
|
|
_failure(
|
|
"INVALID_RESPONSE", "Stability needs both token and content_signature."
|
|
),
|
|
node=last["node"],
|
|
observations=observations,
|
|
attempts=attempts,
|
|
)
|
|
now = monotonic()
|
|
if key != stable_key:
|
|
stable_key, stable_since, stable_samples = key, now, 1
|
|
else:
|
|
stable_samples += 1
|
|
matched = stable_samples >= 2 and (now - stable_since) * 1000 >= request.stable_ms
|
|
else:
|
|
try:
|
|
matched = _condition(request, last["node"])
|
|
except ValueError as exc:
|
|
return _result(
|
|
"wait_until",
|
|
_failure("INVALID_STATE", str(exc)),
|
|
node=last["node"],
|
|
observations=observations,
|
|
attempts=attempts,
|
|
)
|
|
if matched:
|
|
return _result(
|
|
"wait_until",
|
|
{"success": True, "execution": "observed", "code": "OK"},
|
|
native_receipt=last,
|
|
state=request.state,
|
|
node=last.get("node"),
|
|
observations=observations,
|
|
attempts=attempts,
|
|
)
|
|
await _pause(deadline, request.poll_interval_ms / 1000)
|
|
return _result(
|
|
"wait_until",
|
|
_failure("TIMEOUT", "The requested native state was not observed within the deadline."),
|
|
native_receipt=last,
|
|
observations=observations,
|
|
attempts=attempts,
|
|
)
|
|
|
|
|
|
async def _control(request: ControlActionRequest, native: NativeCaller) -> dict:
|
|
inspection = await _inspect(native, {"selector": _selector(request)})
|
|
if not inspection["success"]:
|
|
return _result("control_action", inspection)
|
|
node = inspection["node"]
|
|
condition = None
|
|
if request.ensure_state is not None:
|
|
condition = WaitUntilRequest(
|
|
selector=request.selector,
|
|
timeout_ms=request.verify_timeout_ms,
|
|
**request.ensure_state.model_dump(),
|
|
)
|
|
try:
|
|
satisfied = _condition(condition, node)
|
|
except ValueError as exc:
|
|
return _result(
|
|
"control_action", _failure("INVALID_STATE", str(exc)), preflight=inspection
|
|
)
|
|
if satisfied:
|
|
return _result(
|
|
"control_action", inspection, skipped=True, verification="condition_satisfied"
|
|
)
|
|
invalid = _write_guard(node, request.action, request.expected_token)
|
|
if invalid is None and request.action == "set_progress":
|
|
native_range = _range(node)
|
|
value = request.value
|
|
if (
|
|
native_range is None
|
|
or value is None
|
|
or not math.isfinite(value)
|
|
or not native_range["min"] <= value <= native_range["max"]
|
|
or (native_range.get("type") == 0 and value != int(value))
|
|
):
|
|
invalid = _failure(
|
|
"INVALID_ARGUMENT", "The value is incompatible with the live native range."
|
|
)
|
|
if invalid is not None:
|
|
return _result("control_action", invalid, preflight=inspection, node_before=node)
|
|
args = request.model_dump(
|
|
mode="json", exclude_none=True, exclude={"ensure_state", "verify_timeout_ms"}
|
|
)
|
|
args["expected_token"] = node["token"]
|
|
receipt = await _call(native, "control_action", args)
|
|
if receipt["success"] and condition is not None:
|
|
verification = await _wait(condition, native)
|
|
outcome = dict(receipt)
|
|
if not verification["success"]:
|
|
outcome.update(
|
|
success=False,
|
|
code=verification["code"],
|
|
error="The input completed, but its requested state was not verified. Do not toggle again blindly.",
|
|
)
|
|
return _result(
|
|
"control_action",
|
|
outcome,
|
|
native_receipt=receipt,
|
|
preflight=inspection,
|
|
postcondition=verification,
|
|
node_before=node,
|
|
node_after=verification.get("node"),
|
|
verification="condition_satisfied" if verification["success"] else "condition_not_met",
|
|
)
|
|
return _result(
|
|
"control_action",
|
|
receipt,
|
|
preflight=inspection,
|
|
node_before=receipt.get("node_before", node),
|
|
node_after=receipt.get("node_after"),
|
|
verification="pending_observation",
|
|
)
|
|
|
|
|
|
async def _scroll(request: ScrollToRequest, native: NativeCaller) -> dict:
|
|
started = monotonic()
|
|
deadline = started + request.timeout_ms / 1000
|
|
container_selector = request.container.model_dump(mode="json", exclude_none=True)
|
|
target_args = {
|
|
"selector": request.target.model_dump(mode="json", exclude_none=True),
|
|
"scope": container_selector,
|
|
}
|
|
observations: list[dict] = []
|
|
scroll_results: list[dict] = []
|
|
last_container: dict = {}
|
|
last_target: dict = {}
|
|
|
|
def finish(receipt: dict) -> dict:
|
|
result = dict(receipt)
|
|
if result.get("execution") not in {"unknown", "cancelled"} and any(
|
|
item["native_receipt"]["success"] for item in scroll_results
|
|
):
|
|
result["execution"] = "completed"
|
|
return _result(
|
|
"scroll_to",
|
|
result,
|
|
node=last_target.get("node"),
|
|
container=last_container.get("node"),
|
|
scroll_results=scroll_results,
|
|
observations=observations,
|
|
verification="pending_observation",
|
|
)
|
|
|
|
async def inspect(args: dict) -> dict:
|
|
receipt = await _inspect(native, args, timeout=max(0, deadline - monotonic()))
|
|
_remember(observations, started, receipt)
|
|
if monotonic() > deadline:
|
|
return _failure("TIMEOUT", "The native observation arrived after the search deadline.")
|
|
return receipt
|
|
|
|
def target_visible(receipt: dict) -> bool:
|
|
if receipt["success"]:
|
|
return receipt["node"].get("visible") is True
|
|
return False
|
|
|
|
def target_state(receipt: dict) -> tuple:
|
|
# A target may be outside the bounded container signature sample.
|
|
# Its own visibility and identity must independently remain quiet too.
|
|
return (True, receipt["node"].get("token")) if target_visible(receipt) else (False,)
|
|
|
|
def target_error(receipt: dict) -> dict | None:
|
|
if not receipt["success"] and receipt.get("code") != "NOT_FOUND":
|
|
return receipt
|
|
if receipt["success"] and type(receipt["node"].get("visible")) is not bool:
|
|
return _failure("INVALID_RESPONSE", "The scoped target has no native visibility state.")
|
|
return None
|
|
|
|
for _ in range(request.max_scrolls):
|
|
if monotonic() >= deadline:
|
|
break
|
|
last_container = await inspect({"selector": container_selector})
|
|
if not last_container["success"]:
|
|
return finish(last_container)
|
|
last_target = await inspect(target_args)
|
|
error = target_error(last_target)
|
|
if error is not None:
|
|
return finish(error)
|
|
if target_visible(last_target):
|
|
return finish({"success": True, "execution": "observed", "code": "OK"})
|
|
node = last_container["node"]
|
|
action = f"scroll_{request.direction}"
|
|
invalid = _write_guard(node, action)
|
|
signature = node.get("content_signature")
|
|
if invalid is not None:
|
|
return finish(invalid)
|
|
if not isinstance(signature, str) or not signature:
|
|
return finish(_failure("INVALID_RESPONSE", "The container has no content_signature."))
|
|
if monotonic() >= deadline:
|
|
break
|
|
receipt = await _call(
|
|
native,
|
|
"control_action",
|
|
{"selector": container_selector, "action": action, "expected_token": node["token"]},
|
|
timeout=CONTROL_COMPLETION_SECONDS,
|
|
)
|
|
scroll_results.append({"node_before": node, "native_receipt": receipt})
|
|
if not receipt["success"]:
|
|
if receipt.get("code") == "ACTION_REJECTED" and receipt["execution"] == "not_started":
|
|
return finish(
|
|
_failure("NO_PROGRESS", "The native container rejected further scrolling.")
|
|
)
|
|
return finish(receipt)
|
|
if monotonic() >= deadline:
|
|
break
|
|
|
|
settle_seconds = request.settle_ms / 1000
|
|
# A native scroll accepts the animation; it does not wait for layout.
|
|
# Require a quiet content interval before the next scroll, including when
|
|
# the target briefly enters the viewport during that animation. Also give
|
|
# a delayed first change a short grace rather than reissuing input.
|
|
no_progress_deadline = min(deadline, monotonic() + max(settle_seconds, 0.5))
|
|
stable_since = monotonic()
|
|
last_signature = (signature, target_state(last_target))
|
|
changed = False
|
|
while True:
|
|
if monotonic() >= deadline:
|
|
break
|
|
last_target = await inspect(target_args)
|
|
error = target_error(last_target)
|
|
if error is not None:
|
|
return finish(error)
|
|
last_container = await inspect({"selector": container_selector})
|
|
if not last_container["success"]:
|
|
return finish(last_container)
|
|
current = last_container["node"].get("content_signature")
|
|
if not isinstance(current, str) or not current:
|
|
return finish(
|
|
_failure("INVALID_RESPONSE", "The container has no content_signature.")
|
|
)
|
|
now = monotonic()
|
|
current_state = (current, target_state(last_target))
|
|
if current_state != last_signature:
|
|
changed = True
|
|
stable_since = now
|
|
last_signature = current_state
|
|
if changed and now - stable_since >= settle_seconds - 1e-9:
|
|
break
|
|
if not changed and now >= no_progress_deadline - 1e-9:
|
|
break
|
|
await _pause(deadline, 0.05)
|
|
if monotonic() >= deadline:
|
|
break
|
|
if target_visible(last_target):
|
|
return finish({"success": True, "execution": "completed", "code": "OK"})
|
|
if not changed:
|
|
return finish(
|
|
_failure("NO_PROGRESS", "The container content did not change after scrolling.")
|
|
)
|
|
code = "TIMEOUT" if monotonic() >= deadline else "MAX_SCROLLS"
|
|
return finish(
|
|
_failure(code, "The visible target was not found within the bounded scroll search.")
|
|
)
|
|
|
|
|
|
async def execute_native(
|
|
name: str,
|
|
request: BaseModel | dict,
|
|
width: int,
|
|
height: int,
|
|
call_native: NativeCaller,
|
|
) -> dict:
|
|
"""Execute one validated generic action without retrying any write.
|
|
|
|
Cancellation always propagates. Structured helper receipts remain available
|
|
even when a surrounding wait/scroll condition was not established.
|
|
"""
|
|
try:
|
|
model = NATIVE_REQUESTS.get(name)
|
|
if model is None:
|
|
raise ValueError("Unknown native action.")
|
|
args = request.model_dump(mode="json") if isinstance(request, BaseModel) else request
|
|
parsed = model.model_validate(args)
|
|
if isinstance(parsed, (PinchRequest, TouchGestureRequest)):
|
|
planned = (
|
|
plan_pinch(parsed, width, height)
|
|
if isinstance(parsed, PinchRequest)
|
|
else plan_touch(parsed, width, height)
|
|
)
|
|
else:
|
|
planned = None
|
|
except (ValidationError, TypeError, ValueError) as exc:
|
|
return _result(name, _failure("INVALID_ARGUMENT", str(exc)))
|
|
|
|
if planned is not None:
|
|
operation_id = str(uuid4())
|
|
args = {
|
|
"operation_id": operation_id,
|
|
"frames": planned["frames"],
|
|
"screen_size": planned["screen_size"],
|
|
}
|
|
if planned.get("require_visible"):
|
|
args["require_visible"] = planned["require_visible"]
|
|
receipt = await _call(
|
|
call_native, "touch_gesture", args, timeout=planned["frames"][-1]["at_ms"] / 1000 + 5
|
|
)
|
|
return _result(
|
|
name,
|
|
receipt,
|
|
operation_id=operation_id,
|
|
plan=planned,
|
|
verification="pending_observation",
|
|
)
|
|
if isinstance(parsed, WaitUntilRequest):
|
|
return await _wait(parsed, call_native)
|
|
if isinstance(parsed, ControlActionRequest):
|
|
return await _control(parsed, call_native)
|
|
if isinstance(parsed, ScrollToRequest):
|
|
return await _scroll(parsed, call_native)
|
|
if isinstance(parsed, ReadControlRequest):
|
|
receipt = await _inspect(call_native, {"selector": _selector(parsed)})
|
|
return _result(name, receipt)
|
|
return _result(
|
|
name, _failure("INVALID_ARGUMENT", "The action has no native execution handler.")
|
|
)
|