feat(android): add native controls and continuous multi-touch

This commit is contained in:
chenmingxuan committed 2026-11-02 21:33:45 +08:00
1 parent 3dd96078db
commit 4b12ae7728
70 files changed
+13105 -179

No files matched your search

@@ -0,0 +1,103 @@
"""Guard the prompt evidence contract; no model or device is invoked here."""
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from artemis.agents.checker.checker import (
CheckReport,
assemble_checker_prompt_segments,
run_final_check,
)
from artemis.context import ArtemisContext, ExecutionSetup
@pytest.mark.parametrize("entry", ["checkpoint", "final"])
def test_bound_persistent_readout_can_supply_precision_beyond_pixel_ticks(entry):
prompt = assemble_checker_prompt_segments(entry, [], probe_tool_registered=False)
# Native origin alone is insufficient: the rule must bind the displayed value
# to the selected state and require stability plus the visible pointer check.
for requirement in (
"标签、节点角色/层级",
"目标控件当前所选值或固定指针所选时间",
"不能仅因原生节点含有数值或执行者称其可信",
"在被验收状态中持续显示",
"引用已停稳或暂停的证据",
"与同一控件的指针及可辨刻度一致",
"结合读数精度与用户容差",
"粗刻度无法独立区分秒数,不等于",
"无需再由粗像素刻度独立插值证明同一精确值",
):
assert requirement in prompt
assert "拖动浮层或原生时间读数与固定指针刻度冲突时" not in prompt
@pytest.mark.parametrize("entry", ["checkpoint", "final"])
def test_native_origin_does_not_promote_unbound_or_conflicting_evidence(entry):
prompt = assemble_checker_prompt_segments(entry, [], probe_tool_registered=False)
for excluded_source in ("事件卡片时间", "视频画面水印", "录制起止时间", "拖动/松手浮层"):
assert excluded_source in prompt
assert "即使它们也能通过原生节点读取,仍须先证明上述绑定与持久性" in prompt
assert "可辨精度上存在实际冲突" in prompt
assert "读数仍在变化、绑定不明或无法确认已停稳/暂停" in prompt
assert "应判证据不足(inconclusive)" in prompt
assert "不得挑选有利读数或忽略矛盾" in prompt
assert "可信读数明确超出用户容差时仍按通用规则判定" in prompt
@pytest.mark.parametrize("entry", ["checkpoint", "final"])
def test_pixel_fallback_remains_quantitative_and_requires_post_action_evidence(entry):
prompt = assemble_checker_prompt_segments(entry, [], probe_tool_registered=False)
fallback = prompt.split("没有可信持久读数时,", 1)[1]
assert "动作后已停稳截图的两处真实数值刻度与固定指针位置作线性插值" in fallback
assert "单位换算、估读不确定性及目标误差" in fallback
assert "目标误差加估读不确定性不超过用户容差" in fallback
assert "刻度非线性或无法确定已停稳,仍应判证据不足" in fallback
assert "操作前的校准计划及回执误差不能替代动作后测量" in fallback
@pytest.mark.asyncio
async def test_final_review_receives_live_readout_and_rules_separately_from_report_claims():
ctx = MagicMock(spec=ArtemisContext)
ctx.execution_setup = ExecutionSetup()
ctx.data_engine = MagicMock()
ctx.data_engine.base_dir = "unused"
ctx.data_engine.get_agent_friendly_steps.return_value = []
ctx.data_engine.get_step_number.return_value = 7
elements = "[1] 当前所选值: 23.5; [2] 已暂停"
review = AsyncMock(return_value=CheckReport(verdicts=[]))
with (
patch("artemis.agents.checker.checker._run_check_loop", new=review),
patch(
"artemis.agents.checker.checker._capture_final_screen",
new=AsyncMock(return_value=("captured-image", elements)),
),
):
await run_final_check(
ctx,
goal="核对当前所选值及容差",
plan_text="",
ledger=[],
check_items=[],
execution_report="执行者声称当前所选值为 88",
)
review.assert_awaited_once()
messages = review.await_args.args[1]
assert "无需再由粗像素刻度独立插值证明同一精确值" in messages[0].content
assert "不得挑选有利读数或忽略矛盾" in messages[0].content
content = messages[1].content
text = content[0]["text"]
assert "Executor's Final Report (claims to verify, not independent evidence)" in text
assert "执行者声称当前所选值为 88" in text
assert f"# Final Screen Elements\n{elements}" in text
assert any(
block.get("type") == "image_url"
and block["image_url"]["url"] == "data:image/jpeg;base64,captured-image"
for block in content
)
+194
View File
@@ -0,0 +1,194 @@
"""Actual action-result/history/transcript rendering must keep native evidence."""
from copy import deepcopy
import json
from types import SimpleNamespace
from unittest.mock import AsyncMock
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
import pytest
from artemis.agents.validator.action_execution import exec_action
from artemis.mcp.action_types import ActionCode, ActionResult
from artemis.memory.transcript import EXECUTION_RESULT_MARKER, TranscriptLedger
from artemis.utils.task_tree import format_result_clean
NATIVE_ACTIONS = (
"pinch",
"touch_gesture",
"read_control",
"control_action",
"wait_until",
"scroll_to",
)
TOKEN = "a" * 64
SELECTOR = {"resource_id": "fixture:id/editor"}
def action_arguments(action):
drag = {
"start": [191, 501],
"end": [192, 532],
"coordinate_space": "pixel",
"target_description": "observed canvas",
}
return {
"pinch": {
"center": [200, 400],
"start_span": 100,
"end_span": 200,
"coordinate_space": "pixel",
"target_description": "observed canvas",
},
"touch_gesture": {
"frames": [
{"at_ms": 0, "points": [[191, 501]]},
{"at_ms": 400, "points": [[192, 532]]},
],
"coordinate_space": "pixel",
"target_description": "observed canvas",
},
"read_control": {"selector": SELECTOR},
"control_action": {
"selector": SELECTOR,
"action": "set_selection",
"selection_start": 3,
"selection_end": 5,
},
"wait_until": {"selector": SELECTOR, "state": "value", "expected_value": 37.5},
"scroll_to": {"container": {"resource_id": "fixture:id/list"}, "target": SELECTOR},
"drag": drag,
"action_sequence": {"steps": [{"action": "drag", **drag}]},
}[action]
def control_observation():
return {
"resource_id": "fixture:id/editor",
"text": " A😀B \n",
"selection_start": 3,
"selection_end": 5,
"range": {"min": 0, "max": 100, "current": 37.5, "type": 1},
"token": TOKEN,
"content_signature": "b" * 64,
"editable": True,
"password": False,
}
def decode_receipt(text, label):
marker = f"{label} receipt: "
assert marker in text
return json.loads(text.split(marker, 1)[1])
async def render_result_pipeline(action, metadata, failed):
message = (
"Native reply is uncertain; observe before continuing." if failed else "Input accepted."
)
result = (
ActionResult.failure(action, message, code=ActionCode.TIMEOUT, metadata=metadata)
if failed
else ActionResult.success(action, message, metadata=metadata)
)
session = SimpleNamespace(call=AsyncMock(return_value=result))
context = SimpleNamespace(
data_engine=None, device=SimpleNamespace(device_width=400, device_height=800)
)
args = action_arguments(action)
item = {"action": action, "args": args}
ok, error = await exec_action(context, session, item)
assert ok is not failed
session.call.assert_awaited_once_with(action, args)
assert item["result"] == result.agent_message
report = {
"status": "failed" if failed else "dispatched",
"execution": [{**item, "attempts": [f"Error: {error}" if failed else "Dispatched"]}],
}
history = format_result_clean(report)
assert isinstance(history, str)
ledger = TranscriptLedger(clock=lambda: 0.0)
ledger.stage_turn(
[
HumanMessage(content="Observed native controls"),
AIMessage(content="", tool_calls=[{"name": action, "args": {}, "id": "native-call"}]),
ToolMessage(content="Action Recorded", tool_call_id="native-call"),
]
)
ledger.commit_staged(step_key="native-step", validator_result=report)
rendered = ledger.render([HumanMessage(content="Next observation")])
execution_results = [
message.content[0]["text"]
for message in rendered
if isinstance(message, HumanMessage)
and isinstance(message.content, list)
and message.content
and EXECUTION_RESULT_MARKER in message.content[0].get("text", "")
]
assert len(execution_results) == 1
transcript = execution_results[0]
assert f"Status: {report['status']}" in transcript
if failed:
assert "Error:" in history and "Error:" in transcript
return result.agent_message, history, transcript
@pytest.mark.asyncio
@pytest.mark.parametrize("action", NATIVE_ACTIONS)
@pytest.mark.parametrize("failed", [False, True])
async def test_every_native_action_keeps_readings_selection_range_and_token_in_actual_result_rendering(
action, failed
):
node = control_observation()
read_only = action in {"read_control", "wait_until"}
execution = (
("not_started" if read_only else "unknown")
if failed
else ("observed" if read_only else "completed")
)
# The renderer must retain optional observed evidence even on gesture receipts.
metadata = {
"action": action,
"execution": execution,
"node": deepcopy(node),
"native_receipt": {
"success": not failed,
"execution": execution,
"code": "TIMEOUT" if failed else "OK",
"node": deepcopy(node),
},
"verification": "pending_observation",
}
stages = await render_result_pipeline(action, metadata, failed)
for rendered in stages:
recovered = decode_receipt(rendered, "Native")
assert recovered == metadata
assert recovered["node"]["text"] == " A😀B \n"
assert recovered["node"]["range"]["current"] == 37.5
assert (recovered["node"]["selection_start"], recovered["node"]["selection_end"]) == (3, 5)
assert recovered["node"]["token"] == TOKEN
assert recovered["native_receipt"]["success"] is not failed
@pytest.mark.asyncio
@pytest.mark.parametrize("action", ["drag", "action_sequence"])
@pytest.mark.parametrize("failed", [False, True])
async def test_existing_drag_and_sequence_geometry_and_partial_receipts_still_render(
action, failed
):
metadata = {
"execution": "partial" if failed else "completed",
"coordinate_space": "pixel",
"start": [191, 501],
"end": [192, 532],
"calibration": {"axis": "y", "target_value": 37.5},
}
if action == "action_sequence":
metadata["sequence_results"] = [
{"action": "click", "execution": "completed", "completed_taps": 1},
{"action": "drag", "execution": "unknown" if failed else "completed"},
]
for rendered in await render_result_pipeline(action, metadata, failed):
assert decode_receipt(rendered, "Gesture") == metadata
+39 -2
View File
@@ -35,6 +35,7 @@ from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, Tool
from artemis.agents.operator.operator import OperatorNode
from artemis.agents.operator.prompts import load_operator_prompts
from artemis.agents.operator.prompts import (
OPERATOR_PROMPT_TOOLSET,
PLAN_HISTORY_STATIC_POINTER,
PLAN_HISTORY_TEMPLATE_SECTION,
PromptBuilder,
@@ -54,8 +55,8 @@ from artemis.memory.transcript import (
# SHA-256 snapshots of the legacy system message with the fixed inputs below.
# Update these when an intentional template change alters the rendered prompt.
GOLDEN_EMPTY_PLAN = "23c3d6b5fc3694275e7015f7bd0901bde54aae54fec7a8c68d4f8529c65ef4bc"
GOLDEN_SENTINEL_PLAN = "370c53f1d088a4120c63fbaea6d895f121c683b8d3c8cc7d61105d6e1d223128"
GOLDEN_EMPTY_PLAN = "f5ee50f4923fed4cb943cb80087d39ffffe66f259cfae4a54f27a244d7b06453"
GOLDEN_SENTINEL_PLAN = "bb725e2d096f6aa0cf1773809b2d0e672475589782b9c72348d9eaf8481230b9"
SCREENSHOT_B64 = base64.b64encode(b"fake-jpeg-bytes").decode("utf-8")
@@ -84,6 +85,42 @@ async def test_legacy_prompt_bytes_match_chinese_prompt_golden():
assert hashlib.sha256(sentinel.encode("utf-8")).hexdigest() == GOLDEN_SENTINEL_PLAN
@pytest.mark.asyncio
@pytest.mark.parametrize("transcript", [False, True])
async def test_legacy_and_transcript_prompts_omit_all_unavailable_native_capabilities(transcript):
native_tools = {
"pinch",
"touch_gesture",
"read_control",
"control_action",
"wait_until",
"scroll_to",
}
available_tools = OPERATOR_PROMPT_TOOLSET - native_tools
ctx = SimpleNamespace(
execution_setup=None,
actuator=SimpleNamespace(capabilities=lambda: available_tools),
data_engine=None,
)
state = SimpleNamespace(initial_goal="FILTERED_FIXED_GOAL")
if transcript:
system = render_transcript_static_system(load_operator_prompts(), ctx, state)
else:
builder = PromptBuilder()
await TemplatePromptComponent()(
builder,
state,
ctx,
prompts=load_operator_prompts(),
template_name="main_template",
)
system = "".join(builder.system_parts)
assert "FILTERED_FIXED_GOAL" in system
assert "`click`" in system and "`drag`" in system
for tool in native_tools:
assert tool not in system
def test_main_template_contains_exactly_one_plan_history_section():
template = load_operator_prompts()["main_template"]
assert template.count(PLAN_HISTORY_TEMPLATE_SECTION) == 1
+39 -9
View File
@@ -38,6 +38,14 @@ OPERATOR_DEVICE_TOOLS = (
"wait_for_delay",
"long_press",
)
OPERATOR_NATIVE_TOOLS = (
"pinch",
"touch_gesture",
"read_control",
"control_action",
"wait_until",
"scroll_to",
)
def _tool_refs(text: str, tool: str) -> list[str]:
@@ -88,6 +96,7 @@ def test_full_toolset_is_the_default():
@pytest.mark.parametrize(
"tool",
OPERATOR_DEVICE_TOOLS
+ OPERATOR_NATIVE_TOOLS
+ ("video_analyzer", "search_history", "replay_steps", "get_step_screenshot"),
)
def test_removed_tool_leaves_no_reference(tool):
@@ -99,6 +108,23 @@ def test_removed_tool_leaves_no_reference(tool):
assert not _tool_refs(out, tool), f"'{tool}' still referenced in {name}"
def test_removing_all_native_capabilities_leaves_no_native_names_in_any_template():
available_tools = OPERATOR_PROMPT_TOOLSET - set(OPERATOR_NATIVE_TOOLS)
for name, template in load_operator_prompts().items():
out = apply_operator_prompt_contract(template, available_tools=available_tools)
for tool in OPERATOR_NATIVE_TOOLS:
assert tool not in out, f"'{tool}' still advertised without native capability in {name}"
assert "`click`" in out and "`drag`" in out
assert (
"Physical device actions (`click`, `action_sequence`, `drag`, `input_text`, `swipe`,"
" `press_key`, `manage_app`, `wait_for_delay`, `long_press`)" in out
)
assert (
"Turn-Ending Action (`click`, `action_sequence`, `drag`, `swipe`, `input_text`,"
" `long_press`, `press_key`, `manage_app`, or `wait_for_delay`)" in out
)
def test_removed_swipe_takes_its_teaching_along():
"""Gating removes the drag/slider teaching, not just the backticked name."""
template = load_operator_prompts()["main_template"]
@@ -131,26 +157,30 @@ def test_reduced_enums_stay_well_formed():
available_tools=OPERATOR_PROMPT_TOOLSET - {"manage_app", "wait_for_delay"},
)
assert (
"Physical device actions (`click`, `action_sequence`, `drag`, `input_text`, `swipe`, `press_key`, `long_press`)"
in out
"Physical device actions (`click`, `action_sequence`, `drag`, `pinch`, `touch_gesture`,"
" `read_control`, `control_action`, `wait_until`, `scroll_to`, `input_text`, `swipe`,"
" `press_key`, `long_press`)" in out
)
assert (
"Turn-Ending Action (`click`, `action_sequence`, `drag`, `swipe`, `input_text`, `long_press`, or `press_key`)"
in out
"Turn-Ending Action (`click`, `action_sequence`, `drag`, `pinch`, `touch_gesture`,"
" `read_control`, `control_action`, `wait_until`, `scroll_to`, `swipe`, `input_text`,"
" `long_press`, or `press_key`)" in out
)
def test_full_set_enum_slots_render_verbatim():
"""With every tool present the historical enumeration wording is reproduced."""
"""Every supported tool appears in the exact physical and turn-ending order."""
template = load_operator_prompts()["main_template"]
out = apply_operator_prompt_contract(template)
assert (
"Physical device actions (`click`, `action_sequence`, `drag`, `input_text`, `swipe`, `press_key`,"
" `manage_app`, `wait_for_delay`, `long_press`)" in out
"Physical device actions (`click`, `action_sequence`, `drag`, `pinch`, `touch_gesture`,"
" `read_control`, `control_action`, `wait_until`, `scroll_to`, `input_text`, `swipe`,"
" `press_key`, `manage_app`, `wait_for_delay`, `long_press`)" in out
)
assert (
"Turn-Ending Action (`click`, `action_sequence`, `drag`, `swipe`, `input_text`, `long_press`,"
" `press_key`, `manage_app`, or `wait_for_delay`)" in out
"Turn-Ending Action (`click`, `action_sequence`, `drag`, `pinch`, `touch_gesture`,"
" `read_control`, `control_action`, `wait_until`, `scroll_to`, `swipe`, `input_text`,"
" `long_press`, `press_key`, `manage_app`, or `wait_for_delay`)" in out
)
assert "Helper/Subagent tools (`ask_explorer`, `ask_diagnoser`, `video_analyzer`)" in out
assert "ADB/task tools (`run_adb_command`, `manage_task`, `android_file`)" in out
+153
View File
@@ -0,0 +1,153 @@
"""Native input is one-shot and remains owned through uncertain transport cleanup."""
import json
from types import SimpleNamespace
import urllib.error
from unittest.mock import Mock
import pytest
from artemis.clients import accessibility_client as module
from artemis.clients.accessibility_client import AccessibilityClient
from artemis.clients.screen_client_factory import FallbackScreenClient
from artemis.runtime.helper_manager import HelperSession
from artemis.runtime.native_protocol import input_completion_seconds
@pytest.fixture
def native(monkeypatch):
manager = Mock()
client = AccessibilityClient("test-device", manager=manager)
client._session = HelperSession(
serial="test-device",
local_port=40001,
transport_id="1",
version_code=12,
version_name="1.3.0",
owns_forward=True,
token="test-only",
protocol_version=3,
capabilities=("inspect_control", "control_action", "touch_gesture", "cancel_gesture"),
)
clock = [0.0]
sleeps = []
def sleep(seconds):
sleeps.append(seconds)
clock[0] += seconds
monkeypatch.setattr(module, "time", SimpleNamespace(monotonic=lambda: clock[0], sleep=sleep))
client._http_once = Mock(return_value=b'{"success":true,"execution":"completed","code":"OK"}')
return client, manager, clock, sleeps
def test_exactly_one_request_and_command_cannot_be_overridden(native):
client, manager, _, sleeps = native
receipt = {
"success": True,
"execution": "observed",
"code": "OK",
"node": {"selection_start": 3},
}
client._http_once.return_value = json.dumps(receipt).encode()
assert client.native_request("inspect_control", {"cmd": "swipe"}, timeout=0.02) == receipt
client._http_once.assert_called_once_with(
client.session, "/action", {"cmd": "inspect_control"}, 0.02
)
assert sleeps == []
manager.attach.assert_not_called()
@pytest.mark.parametrize(
"status,execution,attempts,repairs",
[
(400, "not_started", 1, 0),
(401, "not_started", 1, 0),
(403, "not_started", 1, 0),
(503, "unknown", 1, 0),
],
)
def test_http_rejection_never_replays_native_input(native, status, execution, attempts, repairs):
client, manager, _, sleeps = native
client._http_once.side_effect = urllib.error.HTTPError(
"http://test", status, "rejected", {}, None
)
result = client.native_request("control_action", {"action": "click"}, timeout=0.02)
assert result["execution"] == execution and not result["success"]
assert client._http_once.call_count == attempts
assert client._http_once.call_args.args[-1] >= 5
assert manager.push_token.call_count == repairs
manager.reattach.assert_not_called()
assert bool(sleeps) == (execution == "unknown")
@pytest.mark.parametrize(
"response", [b"broken", b"[]", b"{}", b'{"success":true,"execution":"unknown"}']
)
def test_uncertain_write_drains_without_retry(native, response):
client, _, clock, sleeps = native
client._http_once.return_value = response
result = client.native_request("control_action", {"action": "click"})
assert result["execution"] == "unknown"
assert sleeps == [5.0] and clock[0] == 5.0
client._http_once.assert_called_once()
def test_partial_transport_elapsed_time_counts_toward_input_drain(native):
client, _, clock, sleeps = native
def broken(*args):
clock[0] = 2
raise OSError("response lost after dispatch")
client._http_once.side_effect = broken
assert client.native_request("control_action", {})["execution"] == "unknown"
assert sleeps == [3.0] and clock[0] == 5
client._http_once.assert_called_once()
@pytest.mark.parametrize("command", ["inspect_control", "cancel_gesture"])
def test_non_input_calls_do_not_drain_on_failure(native, command):
client, _, _, sleeps = native
client._http_once.side_effect = OSError("offline")
result = client.native_request(command, {}, timeout=0.02)
assert not result["success"] and sleeps == []
client._http_once.assert_called_once()
def test_no_session_or_capability_cannot_trigger_attach_or_input(native):
client, manager, _, sleeps = native
client._session.capabilities = ()
assert client.native_request("touch_gesture", {})["code"] == "UNSUPPORTED"
client._session = None
assert client.native_capabilities == frozenset()
assert client.native_request("control_action", {})["execution"] == "not_started"
client._http_once.assert_not_called()
manager.attach.assert_not_called()
assert sleeps == []
@pytest.mark.parametrize(
"duration,budget", [(0, 5), (1000, 6), (10000, 15), (999999, 15), (None, 15)]
)
def test_touch_completion_budget_is_bounded(duration, budget):
assert input_completion_seconds("touch_gesture", {"frames": [{"at_ms": duration}]}) == budget
@pytest.mark.parametrize("backend", ["helper", "uiautomator"])
def test_native_input_never_uses_fallback_dispatch(backend):
client = object.__new__(FallbackScreenClient)
client._active_backend = backend
client._helper = Mock()
client._helper.native_capabilities = frozenset({"control_action"})
unknown = {"success": False, "execution": "unknown", "code": "TRANSPORT_ERROR"}
client._helper.native_request.return_value = unknown
client._call = Mock(side_effect=AssertionError("fallback is forbidden"))
result = client.native_request("control_action", {})
if backend == "helper":
assert result is unknown and client.native_capabilities == frozenset({"control_action"})
client._helper.native_request.assert_called_once()
else:
assert result["execution"] == "not_started" and client.native_capabilities == frozenset()
client._helper.native_request.assert_not_called()
client._call.assert_not_called()
@@ -0,0 +1,70 @@
"""Cancellation drains input threads, while read-only work cannot hold the deadline."""
import asyncio
from threading import Event
from types import SimpleNamespace
import pytest
from artemis.drivers.android.adb_driver import AndroidAdbDriver
@pytest.mark.asyncio
@pytest.mark.parametrize("command", ["control_action", "touch_gesture", "inspect_control"])
async def test_cancel_keeps_input_owned_until_transport_finishes(command):
started, release, finished, cancel_seen = Event(), Event(), Event(), Event()
def request(name, args, *, timeout):
if name == "cancel_gesture":
cancel_seen.set()
return {"success": True, "execution": "observed", "code": "CANCEL_RECORDED"}
started.set()
try:
if not release.wait(2):
raise RuntimeError("test transport was not released")
return {"success": True, "execution": "completed", "code": "OK"}
finally:
finished.set()
driver = object.__new__(AndroidAdbDriver)
driver._ui_adb_client = SimpleNamespace(native_request=request)
operation = asyncio.create_task(driver.native_request(command, {"operation_id": "test-cancel"}))
try:
assert await asyncio.to_thread(started.wait, 1)
operation.cancel()
done, _ = await asyncio.wait({operation}, timeout=0.05)
if command == "inspect_control":
assert operation in done and not finished.is_set()
else:
assert operation not in done and not finished.is_set()
operation.cancel()
done, _ = await asyncio.wait({operation}, timeout=0.02)
assert not done
assert cancel_seen.is_set() == (command == "touch_gesture")
finally:
release.set()
with pytest.raises(asyncio.CancelledError):
await operation
assert await asyncio.to_thread(finished.wait, 1)
@pytest.mark.asyncio
async def test_cancelled_input_transport_failure_does_not_replace_cancellation():
started, release = Event(), Event()
def request(*args, **kwargs):
started.set()
release.wait(2)
raise OSError("response lost")
driver = object.__new__(AndroidAdbDriver)
driver._ui_adb_client = SimpleNamespace(native_request=request)
operation = asyncio.create_task(driver.native_request("control_action", {}))
try:
assert await asyncio.to_thread(started.wait, 1)
operation.cancel()
await asyncio.sleep(0)
finally:
release.set()
with pytest.raises(asyncio.CancelledError):
await operation
+12
View File
@@ -85,6 +85,12 @@ def test_tool_declarations_match_fixture():
"SWIPE_TOOL": "swipe",
"DRAG_TOOL": "drag",
"ACTION_SEQUENCE_TOOL": "action_sequence",
"PINCH_TOOL": "pinch",
"TOUCH_GESTURE_TOOL": "touch_gesture",
"READ_CONTROL_TOOL": "read_control",
"CONTROL_ACTION_TOOL": "control_action",
"WAIT_UNTIL_TOOL": "wait_until",
"SCROLL_TO_TOOL": "scroll_to",
"PRESS_KEY_TOOL": "press_key",
"MANAGE_APP_TOOL": "manage_app",
"WAIT_FOR_DELAY_TOOL": "wait_for_delay",
@@ -125,6 +131,12 @@ def test_validator_declaration_order_is_stable():
"swipe",
"drag",
"action_sequence",
"pinch",
"touch_gesture",
"read_control",
"control_action",
"wait_until",
"scroll_to",
"press_key",
"read_note",
"list_notes",
+404
View File
@@ -0,0 +1,404 @@
"""Native action input contracts and geometry, without dispatching device input."""
from copy import deepcopy
import math
import pytest
from pydantic import ValidationError
from artemis.mcp.native_actions import (
NATIVE_REQUESTS,
ControlActionRequest,
ControlSelector,
PinchRequest,
TouchGestureRequest,
WaitUntilRequest,
plan_pinch,
plan_touch,
validate_native_action,
)
def touch(**overrides):
return {
"frames": [
{"at_ms": 0, "points": [[100, 200]]},
{"at_ms": 400, "points": [[300, 600]]},
],
"coordinate_space": "pixel",
"target_description": "observed canvas",
**overrides,
}
def pinch(**overrides):
return {
"center": [200, 400],
"start_span": 100,
"end_span": 200,
"coordinate_space": "pixel",
"target_description": "observed canvas",
**overrides,
}
VALID_REQUESTS = {
"pinch": pinch(),
"touch_gesture": touch(),
"read_control": {"selector": {"resource_id": "fixture:id/range"}},
"control_action": {
"selector": {"resource_id": "fixture:id/range"},
"action": "set_progress",
"value": 37.5,
},
"wait_until": {"selector": {"text": "Ready"}, "state": "visible"},
"scroll_to": {
"container": {"resource_id": "fixture:id/list"},
"target": {"text": "Destination"},
},
}
@pytest.mark.parametrize("name", VALID_REQUESTS)
def test_every_native_request_validates_before_dispatch_and_forbids_extra_fields(name):
args = deepcopy(VALID_REQUESTS[name])
assert validate_native_action(name, args, 400, 800).model_dump(exclude_none=True)
with pytest.raises(ValidationError):
validate_native_action(name, {**args, "coordinate_spce": "pixel"}, 400, 800)
@pytest.mark.parametrize(
"selector",
[
{},
{"bounds": [0, 0, 10, 10]},
{"text": ""},
{"text": 123},
{"resource_id": "item", "resourceId": "typo"},
{"resource_id": "item", "bounds": [0, 0, 10]},
{"resource_id": "item", "bounds": [0, 0, 10, 10, 20]},
{"resource_id": "item", "bounds": [10, 0, 10, 20]},
{"resource_id": "item", "bounds": [20, 0, 10, 20]},
{"resource_id": "item", "bounds": [0, 0, 10, 0]},
{"resource_id": "item", "bounds": [False, 0, 10, 10]},
{"resource_id": "item", "bounds": [0, 0, 10.0, 10]},
{"resource_id": "item", "bounds": [0, 0, "10", 10]},
],
)
def test_selectors_require_observed_semantics_and_strict_nonempty_bounds(selector):
with pytest.raises(ValidationError):
ControlSelector.model_validate(selector)
def test_exact_selector_strings_keep_whitespace_and_do_not_choose_one_attribute():
observed = {
"resource_id": "fixture:id/editor",
"text": " A😀B \n",
"content_desc": " description \t",
"class_name": "android.widget.EditText",
"package_name": "fixture",
"bounds": [-5, 20, 100, 60],
}
assert ControlSelector.model_validate(observed).model_dump(exclude_none=True) == observed
@pytest.mark.parametrize(
"frames",
[
[{"at_ms": 0, "points": [[1, 2]]}],
[{"at_ms": i, "points": [[1, 2]]} for i in range(33)],
[{"at_ms": 1, "points": [[1, 2]]}, {"at_ms": 2, "points": [[2, 3]]}],
[{"at_ms": 0, "points": [[1, 2]]}, {"at_ms": 0, "points": [[2, 3]]}],
[{"at_ms": 0, "points": [[1, 2]]}, {"at_ms": -1, "points": [[2, 3]]}],
[{"at_ms": 0, "points": [[1, 2]]}, {"at_ms": 10001, "points": [[2, 3]]}],
[{"at_ms": 0, "points": []}, {"at_ms": 10, "points": []}],
[{"at_ms": 0, "points": [[1, 2]] * 6}, {"at_ms": 10, "points": [[2, 3]] * 6}],
[{"at_ms": 0, "points": [[1, 2]]}, {"at_ms": 10, "points": [[2, 3], [4, 5]]}],
[{"at_ms": 0, "points": [[1]]}, {"at_ms": 10, "points": [[2, 3]]}],
[{"at_ms": 0, "points": [[1, 2, 3]]}, {"at_ms": 10, "points": [[2, 3]]}],
[{"at_ms": 0, "points": [[True, 2]]}, {"at_ms": 10, "points": [[2, 3]]}],
[{"at_ms": 0, "points": [[1.0, 2]]}, {"at_ms": 10, "points": [[2, 3]]}],
[{"at_ms": 0, "points": [["1", 2]]}, {"at_ms": 10, "points": [[2, 3]]}],
[{"at_ms": 0, "points": [[1, 2]]}, {"at_ms": "10", "points": [[2, 3]]}],
[{"at_ms": 0, "points": [[1, 2]]}, {"at_ms": 10.0, "points": [[2, 3]]}],
[{"at_ms": 0, "points": [[1, 2]]}, {"at_ms": True, "points": [[2, 3]]}],
[
{"at_ms": 0, "points": [[1, 2]], "pressure": 0.5},
{"at_ms": 10, "points": [[2, 3]]},
],
],
)
def test_touch_timelines_and_pointer_shapes_are_strict(frames):
with pytest.raises(ValidationError):
TouchGestureRequest.model_validate(touch(frames=frames))
def test_pixel_keyframes_preserve_holds_and_every_intermediate_segment():
frames = [
{"at_ms": 0, "points": [[191, 501], [211, 502]]},
{"at_ms": 150, "points": [[191, 501], [211, 502]]},
{"at_ms": 900, "points": [[192, 532], [212, 533]]},
]
planned = plan_touch(TouchGestureRequest.model_validate(touch(frames=frames)), 400, 800)
assert planned["frames"] == frames
assert planned["coordinate_space"] == "pixel"
assert planned["screen_size"] == [400, 800]
def test_full_supported_pointer_count_frame_count_and_duration_remain_valid():
frames = [
{
"at_ms": round(frame * 10000 / 31),
"points": [[50 + finger * 30, 100 + frame] for finger in range(5)],
}
for frame in range(32)
]
request = validate_native_action("touch_gesture", touch(frames=frames), 400, 800)
assert plan_touch(request, 400, 800)["frames"] == frames
def test_normalized_points_use_each_display_axis_and_accept_the_nominal_edges():
request = TouchGestureRequest.model_validate(
touch(
coordinate_space="normalized",
frames=[
{"at_ms": 0, "points": [[0, 0]]},
{"at_ms": 100, "points": [[500, 500]]},
{"at_ms": 200, "points": [[1000, 1000]]},
],
)
)
assert [f["points"] for f in plan_touch(request, 720, 1600)["frames"]] == [
[[0, 0]],
[[360, 800]],
[[719, 1599]],
]
@pytest.mark.parametrize(
"space,point",
[
("pixel", [-1, 0]),
("pixel", [400, 0]),
("pixel", [0, 800]),
("normalized", [-1, 500]),
("normalized", [1001, 500]),
("normalized", [500, 1001]),
],
)
def test_offscreen_paths_fail_before_any_coordinate_can_be_clamped(space, point):
args = touch(
coordinate_space=space,
frames=[{"at_ms": 0, "points": [[100, 100]]}, {"at_ms": 100, "points": [point]}],
)
with pytest.raises(ValueError, match="outside"):
validate_native_action("touch_gesture", args, 400, 800)
@pytest.mark.parametrize("dimensions", [(0, 800), (400, 0), (-1, 800)])
def test_invalid_display_dimensions_cannot_produce_a_plan(dimensions):
with pytest.raises(ValueError, match="positive"):
validate_native_action("touch_gesture", touch(), *dimensions)
@pytest.mark.parametrize(
"before,after",
[
([[100, 100], [300, 100]], [[300, 100], [100, 100]]),
([[100, 100], [300, 100]], [[200, 200], [200, 200]]),
([[100, 100], [100, 100]], [[100, 200], [300, 200]]),
],
)
def test_fingers_may_not_merge_at_frames_or_between_frames(before, after):
with pytest.raises(ValueError, match="remain distinct"):
validate_native_action(
"touch_gesture",
touch(frames=[{"at_ms": 0, "points": before}, {"at_ms": 100, "points": after}]),
400,
800,
)
def test_paths_can_cross_geometrically_when_fingers_reach_the_crossing_at_different_times():
args = touch(
frames=[
{"at_ms": 0, "points": [[0, 100], [100, 0]]},
{"at_ms": 400, "points": [[200, 100], [100, 400]]},
]
)
assert (
plan_touch(TouchGestureRequest.model_validate(args), 400, 800)["frames"] == args["frames"]
)
@pytest.mark.parametrize("dimensions", [(720, 1600), (1600, 720)])
def test_normalized_pinch_spans_use_short_edge_without_stretching_diagonal_angles(dimensions):
width, height = dimensions
request = PinchRequest.model_validate(
pinch(
center=[500, 500],
start_span=200,
end_span=400,
angle_degrees=45,
coordinate_space="normalized",
)
)
planned = plan_pinch(request, width, height)
for frame, expected_span in zip(planned["frames"], [144, 288], strict=True):
first, second = frame["points"]
assert second[0] - first[0] == second[1] - first[1]
assert math.dist(first, second) == pytest.approx(expected_span, abs=1.5)
assert [(first[i] + second[i]) / 2 for i in (0, 1)] == [width / 2, height / 2]
@pytest.mark.parametrize("rotation", [-360, -180, -90, 90, 180, 360])
def test_rotation_uses_curved_synchronized_keyframes_and_keeps_both_fingers_down(rotation):
planned = plan_pinch(
PinchRequest.model_validate(
pinch(center=[500, 500], start_span=200, end_span=200, rotation_degrees=rotation)
),
1000,
1000,
)
frames = planned["frames"]
assert len(frames) == abs(rotation) // 15 + 1
assert frames[0]["points"] == [[400, 500], [600, 500]]
assert frames[0]["at_ms"] == 0 and frames[-1]["at_ms"] == 800
assert all(a["at_ms"] < b["at_ms"] for a, b in zip(frames, frames[1:]))
assert all(len(frame["points"]) == 2 for frame in frames)
quarter_turn = frames[6]["points"][1]
assert quarter_turn == [500, 600 if rotation > 0 else 400]
for frame in frames:
assert math.dist(*frame["points"]) == pytest.approx(200, abs=1.5)
@pytest.mark.parametrize(
"overrides",
[
{"start_span": 0},
{"end_span": -1},
{"start_span": True},
{"end_span": "200"},
{"end_span": 100},
{"rotation_degrees": 361},
{"angle_degrees": float("nan")},
{"angle_degrees": float("inf")},
{"duration_ms": 0},
{"duration_ms": 10001},
{"duration_ms": 1.0},
{"duration_ms": 20, "rotation_degrees": 360},
{"center": [0, 0]},
{"coordinate_space": "screen"},
],
)
def test_invalid_pinch_geometry_and_units_fail_during_planning(overrides):
with pytest.raises(ValueError):
validate_native_action("pinch", pinch(**overrides), 400, 800)
@pytest.mark.parametrize("action", ["set_text", "replace_selection"])
@pytest.mark.parametrize("text", ["", " A😀B \n\t"])
def test_precise_text_edits_preserve_empty_input_unicode_and_whitespace(action, text):
request = ControlActionRequest.model_validate(
{"selector": {"resource_id": "editor"}, "action": action, "text": text}
)
assert request.model_dump(mode="json", exclude_none=True)["text"] == text
def test_selection_offsets_are_preserved_as_utf16_units_without_python_character_conversion():
text = "A😀B"
end_of_emoji = len(text[:2].encode("utf-16-le")) // 2
request = ControlActionRequest.model_validate(
{
"selector": {"text": text},
"action": "set_selection",
"selection_start": 1,
"selection_end": end_of_emoji,
}
)
wire = request.model_dump(mode="json", exclude_none=True)
assert (wire["selection_start"], wire["selection_end"]) == (1, 3)
# Bounds and surrogate-pair boundaries require the fresh native text, checked by Java.
assert end_of_emoji != len(text[:2])
@pytest.mark.parametrize(
"args",
[
{"action": "click", "value": 1},
{"action": "click", "text": "extra"},
{"action": "click", "selection_start": 0, "selection_end": 1},
{"action": "set_text"},
{"action": "set_text", "text": "x" * 65537},
{"action": "replace_selection"},
{"action": "set_progress"},
{"action": "set_progress", "value": True},
{"action": "set_progress", "value": "3"},
{"action": "set_progress", "value": float("nan")},
{"action": "set_progress", "value": float("inf")},
{"action": "set_selection", "selection_start": 0},
{"action": "set_selection", "selection_start": -1, "selection_end": 1},
{"action": "set_selection", "selection_start": 2, "selection_end": 1},
{"action": "set_selection", "selection_start": False, "selection_end": 1},
{"action": "set_selection", "selection_start": 0, "selection_end": 1.0},
{"action": "set_selection", "selection_start": 0, "selection_end": "1"},
{"action": "click", "expected_token": "a" * 63},
{"action": "click", "expected_token": "G" * 64},
{"action": "toggle"},
],
)
def test_control_actions_reject_coercion_wrong_arguments_and_invalid_tokens(args):
with pytest.raises(ValidationError):
ControlActionRequest.model_validate({"selector": {"resource_id": "control"}, **args})
@pytest.mark.parametrize(
"state,extra",
[
("visible", {}),
("absent", {}),
("enabled", {}),
("disabled", {}),
("checked", {}),
("unchecked", {}),
("selected", {}),
("unselected", {}),
("text", {"expected_text": " exact 😀 \n"}),
("value", {"expected_value": 0}),
],
)
def test_nonstable_waits_allow_short_deadlines_without_overriding_stability_default(state, extra):
request = WaitUntilRequest.model_validate(
{"selector": {"resource_id": "control"}, "state": state, "timeout_ms": 100, **extra}
)
assert request.timeout_ms == 100 and request.stable_ms == 300
if state == "text":
assert request.expected_text == extra["expected_text"]
def test_stability_window_must_fit_the_deadline_only_for_stable_waits():
args = {"selector": {"resource_id": "control"}, "state": "stable", "timeout_ms": 100}
with pytest.raises(ValidationError, match="stable_ms must fit"):
WaitUntilRequest.model_validate(args)
assert WaitUntilRequest.model_validate({**args, "stable_ms": 100}).stable_ms == 100
@pytest.mark.parametrize(
"name,args",
[
("wait_until", {"state": "text"}),
("wait_until", {"state": "value"}),
("wait_until", {"state": "visible", "expected_text": "wrong predicate"}),
("wait_until", {"state": "visible", "tolerance": 1}),
("wait_until", {"state": "value", "expected_value": float("inf")}),
("wait_until", {"state": "visible", "timeout_ms": 0}),
("wait_until", {"state": "visible", "poll_interval_ms": 49}),
("scroll_to", {"max_scrolls": 0}),
("scroll_to", {"max_scrolls": 21}),
("scroll_to", {"timeout_ms": 100, "settle_ms": 200}),
("scroll_to", {"direction": "diagonal"}),
],
)
def test_condition_and_scroll_requests_reject_incompatible_or_unbounded_arguments(name, args):
with pytest.raises(ValidationError):
NATIVE_REQUESTS[name].model_validate({**VALID_REQUESTS[name], **args})
+775
View File
@@ -0,0 +1,775 @@
"""Native semantics with deterministic clocks and no devices or paid models."""
import asyncio
from copy import deepcopy
from unittest.mock import AsyncMock
from uuid import UUID
import pytest
from artemis.mcp import native_executor
from artemis.mcp.native_executor import execute_native
SELECTOR = {"resource_id": "control"}
CONTAINER = {"resource_id": "list"}
TARGET = {"text": "destination"}
TOKEN = "a" * 64
def node(**overrides):
return {
"token": TOKEN,
"content_signature": "contents-a",
"visible": True,
"enabled": True,
"checkable": True,
"checked": False,
"selected": False,
"password": False,
"text": "exact text",
"actions": ["click", "set_progress", "set_text", "scroll_forward"],
"range": {"min": 0, "max": 10, "current": 3, "type": 1},
**overrides,
}
def observed(**overrides):
return {"success": True, "execution": "observed", "code": "OK", "node": node(**overrides)}
def missing(code="NOT_FOUND"):
return {"success": False, "execution": "not_started", "code": code, "error": code}
def completed(**extra):
return {"success": True, "execution": "completed", "code": "OK", **extra}
class Clock:
def __init__(self):
self.now = 0.0
self.sleeps = []
def monotonic(self):
return self.now
async def sleep(self, seconds):
assert seconds > 0
self.sleeps.append(seconds)
self.now += seconds
@pytest.fixture
def clock(monkeypatch):
fake = Clock()
monkeypatch.setattr(native_executor, "monotonic", fake.monotonic)
monkeypatch.setattr(native_executor, "sleep", fake.sleep)
return fake
async def run(name, args, native):
return await execute_native(name, args, 400, 800, native)
@pytest.mark.asyncio
@pytest.mark.parametrize("name", ["pinch", "touch_gesture"])
async def test_all_gesture_geometry_is_validated_before_a_single_native_dispatch(name):
args = (
{
"center": [200, 400],
"start_span": 40,
"end_span": 80,
"rotation_degrees": 45,
"duration_ms": 900,
"coordinate_space": "pixel",
"target_description": "observed control",
}
if name == "pinch"
else {
"frames": [
{"at_ms": 0, "points": [[100, 100]]},
{"at_ms": 300, "points": [[120, 160]]},
{"at_ms": 900, "points": [[140, 220]]},
],
"coordinate_space": "pixel",
"target_description": "observed control",
}
)
native = AsyncMock(return_value=completed(native_marker="original receipt"))
result = await run(name, args, native)
assert result["success"] and result["execution"] == "completed"
assert result["verification"] == "pending_observation"
assert result["native_receipt"]["native_marker"] == "original receipt"
native.assert_awaited_once()
command, sent = native.await_args.args
assert command == "touch_gesture"
assert UUID(sent["operation_id"])
assert sent["frames"] == result["plan"]["frames"]
assert sent["screen_size"] == [400, 800]
assert native.await_args.kwargs == {"timeout": 5.9}
invalid = deepcopy(args)
if name == "pinch":
invalid["center"] = [0, 0]
else:
invalid["frames"][-1]["points"] = [[401, 220]]
never = AsyncMock()
refused = await run(name, invalid, never)
assert not refused["success"] and refused["execution"] == "not_started"
never.assert_not_awaited()
@pytest.mark.asyncio
async def test_touch_timeout_never_replays_the_input():
native = AsyncMock(side_effect=TimeoutError("reply lost"))
result = await run(
"touch_gesture",
{
"frames": [{"at_ms": 0, "points": [[10, 10]]}, {"at_ms": 20, "points": [[20, 30]]}],
"target_description": "observed control",
"require_visible": "required control",
},
native,
)
assert not result["success"] and result["execution"] == "unknown"
assert result["code"] == "TIMEOUT"
native.assert_awaited_once()
assert native.await_args.args[1]["require_visible"] == "required control"
@pytest.mark.asyncio
async def test_read_control_keeps_native_evidence_and_omits_null_selector_fields():
receipt = observed()
native = AsyncMock(return_value=receipt)
result = await run("read_control", {"selector": {**SELECTOR, "text": None}}, native)
native.assert_awaited_once_with("inspect_control", {"selector": SELECTOR}, timeout=None)
assert result["node"] == receipt["node"]
assert result["native_receipt"] == receipt
assert result["execution"] == "observed" and result["action"] == "read_control"
@pytest.mark.asyncio
async def test_control_action_uses_fresh_token_preserves_whitespace_and_does_not_verify_business_value():
before = observed()
after = node(text="different accepted text")
receipt = completed(node_before=before["node"], node_after=after)
native = AsyncMock(side_effect=[before, receipt])
result = await run(
"control_action", {"selector": SELECTOR, "action": "set_text", "text": " exact\n"}, native
)
assert native.await_args_list[1].args == (
"control_action",
{"selector": SELECTOR, "action": "set_text", "text": " exact\n", "expected_token": TOKEN},
)
assert native.await_count == 2
assert result["success"] and result["verification"] == "pending_observation"
assert result["node_before"] == before["node"]
assert result["node_after"] == after
assert result["native_receipt"] == receipt
@pytest.mark.asyncio
@pytest.mark.parametrize(
"snapshot,request_extra,code",
[
(node(), {"expected_token": "b" * 64}, "STALE"),
(node(actions=["long_click"]), {}, "UNSUPPORTED"),
(node(enabled=False), {}, "DISABLED"),
(node(token=None), {}, "INVALID_RESPONSE"),
],
)
async def test_control_action_refuses_stale_unsupported_or_disabled_without_input(
snapshot, request_extra, code
):
native = AsyncMock(return_value={**observed(), "node": snapshot})
result = await run(
"control_action", {"selector": SELECTOR, "action": "click", **request_extra}, native
)
assert not result["success"] and result["code"] == code
assert result["execution"] == "not_started"
assert result["node_before"] == snapshot
native.assert_awaited_once()
@pytest.mark.asyncio
@pytest.mark.parametrize(
"native_range,value",
[
({"min": 0, "max": 10, "current": 3}, 11),
({"min": 0, "max": 10, "current": float("nan")}, 3),
({"min": 0, "max": 10, "current": 3, "type": 0}, 3.5),
],
)
async def test_set_progress_refuses_invalid_live_range_before_input(native_range, value):
native = AsyncMock(return_value=observed(range=native_range))
result = await run(
"control_action", {"selector": SELECTOR, "action": "set_progress", "value": value}, native
)
assert not result["success"] and result["execution"] == "not_started"
native.assert_awaited_once()
@pytest.mark.asyncio
async def test_control_transport_failure_is_unknown_and_never_replayed():
native = AsyncMock(side_effect=[observed(), OSError("lost reply")])
result = await run("control_action", {"selector": SELECTOR, "action": "click"}, native)
assert result["code"] == "NATIVE_ERROR" and result["execution"] == "unknown"
assert not result["success"] and native.await_count == 2
@pytest.mark.asyncio
@pytest.mark.parametrize(
"receipt",
[
"legacy unstructured success",
None,
{"success": True, "execution": [], "code": "OK"},
{"success": True, "execution": "unknown", "code": "OK"},
{"success": True, "execution": "completed"},
],
)
async def test_malformed_write_receipts_are_unknown_and_never_replayed(receipt):
native = AsyncMock(side_effect=[observed(), receipt])
result = await run("control_action", {"selector": SELECTOR, "action": "click"}, native)
assert not result["success"] and result["execution"] == "unknown"
assert result["code"] == "INVALID_RESPONSE" and native.await_count == 2
assert result["native_receipt"] == receipt
@pytest.mark.asyncio
async def test_wait_observes_delayed_condition_without_operating_or_sleeping_after_success(clock):
native = AsyncMock(side_effect=[missing(), observed(visible=False), observed()])
result = await run(
"wait_until",
{"selector": SELECTOR, "state": "visible", "timeout_ms": 1000, "poll_interval_ms": 100},
native,
)
assert result["success"] and result["execution"] == "observed"
assert result["attempts"] == 3 and clock.now == pytest.approx(0.2)
assert [call.args[0] for call in native.await_args_list] == ["inspect_control"] * 3
assert [call.kwargs["timeout"] for call in native.await_args_list] == pytest.approx(
[1, 0.9, 0.8]
)
@pytest.mark.asyncio
@pytest.mark.parametrize("changed_field", ["token", "content_signature"])
async def test_stability_restarts_on_identity_or_descendant_redraw(clock, changed_field):
change = {changed_field: "changed"}
native = AsyncMock(
side_effect=[observed(), observed(**change), observed(**change), observed(**change)]
)
result = await run(
"wait_until",
{
"selector": SELECTOR,
"state": "stable",
"stable_ms": 190,
"timeout_ms": 1000,
"poll_interval_ms": 100,
},
native,
)
assert result["success"] and result["attempts"] == 4
assert clock.now == pytest.approx(0.3)
@pytest.mark.asyncio
async def test_zero_stability_window_still_requires_two_actual_reads(clock):
native = AsyncMock(return_value=observed())
result = await run(
"wait_until", {"selector": SELECTOR, "state": "stable", "stable_ms": 0}, native
)
assert result["success"] and result["attempts"] == 2
assert len(clock.sleeps) == 1
@pytest.mark.asyncio
@pytest.mark.parametrize("receipt", [missing(), observed(visible=False)])
async def test_absent_accepts_only_not_found_or_unique_explicitly_invisible(clock, receipt):
native = AsyncMock(return_value=receipt)
result = await run("wait_until", {"selector": SELECTOR, "state": "absent"}, native)
assert result["success"] and result["execution"] == "observed"
native.assert_awaited_once()
@pytest.mark.asyncio
@pytest.mark.parametrize(
"error", ["AMBIGUOUS", "UNSUPPORTED", "LIMIT_EXCEEDED", "INVALID_RESPONSE"]
)
async def test_absent_rejects_nontransient_inspection_errors_immediately(clock, error):
native = AsyncMock(return_value=missing(error))
result = await run("wait_until", {"selector": SELECTOR, "state": "absent"}, native)
assert not result["success"] and result["code"] == error
native.assert_awaited_once()
assert not clock.sleeps
@pytest.mark.asyncio
@pytest.mark.parametrize("error", ["UNAVAILABLE", "READ_FAILED"])
async def test_unreadable_windows_time_out_without_proving_absence(clock, error):
native = AsyncMock(return_value=missing(error))
result = await run(
"wait_until",
{"selector": SELECTOR, "state": "absent", "timeout_ms": 300, "poll_interval_ms": 100},
native,
)
assert not result["success"] and result["code"] == "TIMEOUT"
assert result["native_receipt"]["code"] == error
assert result["attempts"] == 3 and clock.now == pytest.approx(0.3)
assert [sample["code"] for sample in result["observations"]] == [error] * 3
assert [call.args[0] for call in native.await_args_list] == ["inspect_control"] * 3
@pytest.mark.asyncio
@pytest.mark.parametrize("error", ["UNAVAILABLE", "READ_FAILED"])
@pytest.mark.parametrize("state,receipt", [("visible", observed()), ("absent", missing())])
async def test_wait_recovers_only_after_a_reliable_observation(clock, error, state, receipt):
native = AsyncMock(side_effect=[missing(error), receipt])
result = await run(
"wait_until",
{"selector": SELECTOR, "state": state, "timeout_ms": 500, "poll_interval_ms": 100},
native,
)
assert result["success"] and result["execution"] == "observed"
assert result["attempts"] == 2 and clock.now == pytest.approx(0.1)
assert result["observations"][0]["code"] == error
assert result["native_receipt"] == receipt
assert [call.args[0] for call in native.await_args_list] == ["inspect_control"] * 2
@pytest.mark.asyncio
@pytest.mark.parametrize("error", ["UNAVAILABLE", "READ_FAILED"])
async def test_transient_read_failure_restarts_the_full_stability_window(clock, error):
native = AsyncMock(side_effect=[observed(), missing(error), observed(), observed(), observed()])
result = await run(
"wait_until",
{
"selector": SELECTOR,
"state": "stable",
"stable_ms": 190,
"timeout_ms": 1000,
"poll_interval_ms": 100,
},
native,
)
assert result["success"] and result["attempts"] == 5
assert clock.now == pytest.approx(0.4)
assert result["observations"][1]["code"] == error
@pytest.mark.asyncio
async def test_absent_transport_failure_is_not_absence(clock):
native = AsyncMock(side_effect=OSError("unavailable"))
result = await run("wait_until", {"selector": SELECTOR, "state": "absent"}, native)
assert not result["success"] and result["code"] == "NATIVE_ERROR"
native.assert_awaited_once()
@pytest.mark.asyncio
@pytest.mark.parametrize("receipt", [{**observed(), "candidate_count": 2}, observed(visible=None)])
async def test_absent_rejects_nonunique_or_unknown_visibility_observation(clock, receipt):
native = AsyncMock(return_value=receipt)
result = await run("wait_until", {"selector": SELECTOR, "state": "absent"}, native)
assert not result["success"]
assert result["code"] in {"AMBIGUOUS", "INVALID_STATE"}
native.assert_awaited_once()
@pytest.mark.asyncio
@pytest.mark.parametrize(
"state,args,snapshot,success",
[
("text", {"expected_text": " exact text "}, node(), False),
("text", {"expected_text": "exact text"}, node(), True),
("value", {"expected_value": 3.1, "tolerance": 0.2}, node(), True),
(
"value",
{"expected_value": 3},
node(range={"min": 0, "max": float("inf"), "current": 3}),
False,
),
("unchecked", {}, node(checkable=False, checked=False), False),
("checked", {}, node(checked=True), True),
("disabled", {}, node(enabled=False), True),
("unselected", {}, node(selected=False), True),
],
)
async def test_wait_predicates_use_meaningful_exact_native_values(
clock, state, args, snapshot, success
):
native = AsyncMock(return_value={**observed(), "node": snapshot})
result = await run(
"wait_until",
{"selector": SELECTOR, "state": state, "timeout_ms": 400, "stable_ms": 0, **args},
native,
)
assert result["success"] is success
assert all(call.args[0] == "inspect_control" for call in native.await_args_list)
@pytest.mark.asyncio
async def test_wait_deadline_is_bounded_and_preserves_observations(clock):
native = AsyncMock(return_value=observed(visible=False))
result = await run(
"wait_until",
{"selector": SELECTOR, "state": "visible", "timeout_ms": 350, "poll_interval_ms": 100},
native,
)
assert not result["success"] and result["code"] == "TIMEOUT"
assert clock.now == pytest.approx(0.35) and native.await_count == 4
assert len(result["observations"]) == 4
@pytest.mark.asyncio
async def test_wait_rejects_condition_returned_after_monotonic_deadline(clock):
async def native(command, args, *, timeout=None):
clock.now += 0.6
return observed()
result = await run(
"wait_until", {"selector": SELECTOR, "state": "visible", "timeout_ms": 500}, native
)
assert not result["success"] and result["code"] == "TIMEOUT"
assert result["attempts"] == 1
@pytest.mark.asyncio
async def test_wait_stops_a_nonreturning_native_read_at_its_deadline(clock):
calls = []
async def native(command, args, *, timeout=None):
calls.append(command)
await asyncio.Event().wait()
result = await run(
"wait_until",
{"selector": SELECTOR, "state": "visible", "timeout_ms": 20, "stable_ms": 0},
native,
)
assert not result["success"] and result["code"] == "TIMEOUT"
assert calls == ["inspect_control"]
@pytest.mark.asyncio
async def test_long_wait_bounds_evidence_memory_while_retaining_first_and_last_reads(clock):
native = AsyncMock(return_value=observed(visible=False))
result = await run(
"wait_until",
{"selector": SELECTOR, "state": "visible", "timeout_ms": 4000, "poll_interval_ms": 50},
native,
)
assert result["attempts"] > 64 and len(result["observations"]) == 64
assert result["observations"][0]["at_ms"] == 0
assert result["observations"][-1]["at_ms"] > 3000
class ScrollNative:
def __init__(self, clock, *, delayed_target=None, progress=True, receipt=None):
self.clock = clock
self.delayed_target = delayed_target
self.progress = progress
self.receipt = receipt or completed()
self.calls = []
self.writes = 0
async def __call__(self, command, args, *, timeout=None):
self.calls.append((command, deepcopy(args), timeout, self.clock.now))
if command == "control_action":
self.writes += 1
return self.receipt
if args["selector"] == TARGET:
# An unscoped query would spuriously find a same-text external node.
if args.get("scope") != CONTAINER:
return observed(text="external false positive")
if (
self.writes
and self.delayed_target is not None
and self.clock.now >= self.delayed_target
):
return observed(text="destination")
return missing()
assert args == {"selector": CONTAINER}
changed = self.writes and self.progress and self.clock.now >= 0.1
return observed(
token=("b" if self.writes else "a") * 64,
content_signature=f"page-{self.writes}" if changed else "contents-a",
)
def scroll_request(**extra):
return {"container": CONTAINER, "target": TARGET, "settle_ms": 300, "timeout_ms": 2000, **extra}
@pytest.mark.asyncio
async def test_scroll_waits_for_delayed_scoped_target_after_content_changes(clock):
native = ScrollNative(clock, delayed_target=0.2)
result = await run("scroll_to", scroll_request(), native)
assert result["success"] and result["execution"] == "completed"
assert native.writes == 1 and clock.now >= 0.4 - 1e-9
assert result["node"]["text"] == "destination"
assert result["scroll_results"][0]["native_receipt"]["success"]
queries = [
args
for command, args, _, _ in native.calls
if command == "inspect_control" and args["selector"] == TARGET
]
assert queries and all(query["scope"] == CONTAINER for query in queries)
@pytest.mark.asyncio
async def test_scroll_token_change_without_content_change_is_no_progress(clock):
native = ScrollNative(clock, progress=False)
result = await run("scroll_to", scroll_request(), native)
assert not result["success"] and result["code"] == "NO_PROGRESS"
assert result["execution"] == "completed" and native.writes == 1
assert clock.now == pytest.approx(0.5)
assert result["scroll_results"][0]["native_receipt"]["success"]
@pytest.mark.asyncio
async def test_scroll_preserves_each_receipt_and_refreshes_token_with_hard_input_limit(clock):
native = ScrollNative(clock)
result = await run("scroll_to", scroll_request(max_scrolls=2), native)
assert result["code"] == "MAX_SCROLLS" and not result["success"]
assert native.writes == 2 and len(result["scroll_results"]) == 2
writes = [(args, at) for command, args, _, at in native.calls if command == "control_action"]
assert writes[0][0]["expected_token"] == TOKEN
assert writes[1][0]["expected_token"] == "b" * 64
assert writes[1][1] >= 0.3
@pytest.mark.asyncio
@pytest.mark.parametrize("execution", ["unknown", "cancelled"])
async def test_scroll_unknown_or_cancelled_input_is_never_repeated(clock, execution):
receipt = {"success": False, "execution": execution, "code": "TIMEOUT", "native_marker": "kept"}
native = ScrollNative(clock, receipt=receipt)
result = await run("scroll_to", scroll_request(), native)
assert not result["success"] and result["execution"] == execution
assert result["scroll_results"][0]["native_receipt"] == receipt
assert native.writes == 1 and not clock.sleeps
@pytest.mark.asyncio
async def test_scroll_boundary_rejection_is_no_progress_without_retry(clock):
native = ScrollNative(clock, receipt=missing("ACTION_REJECTED"))
result = await run("scroll_to", scroll_request(), native)
assert result["code"] == "NO_PROGRESS" and native.writes == 1
assert result["scroll_results"][0]["native_receipt"]["code"] == "ACTION_REJECTED"
@pytest.mark.asyncio
@pytest.mark.parametrize("which", ["container", "target"])
async def test_scroll_refuses_ambiguous_container_or_scoped_target_before_input(clock, which):
calls = []
async def native(command, args, *, timeout=None):
calls.append(command)
assert command == "inspect_control"
if args["selector"] == (CONTAINER if which == "container" else TARGET):
return missing("AMBIGUOUS")
return observed()
result = await run("scroll_to", scroll_request(), native)
assert not result["success"] and result["code"] == "AMBIGUOUS"
assert "control_action" not in calls
@pytest.mark.asyncio
@pytest.mark.parametrize(
"write_receipt,execution",
[
(completed(native_marker="late accepted input"), "completed"),
(
{
"success": False,
"execution": "unknown",
"code": "TIMEOUT",
"native_marker": "late uncertain receipt",
},
"unknown",
),
],
)
async def test_scroll_drains_dispatched_input_past_search_deadline_without_more_work(
clock, write_receipt, execution
):
calls = []
events = []
pending_input = None
async def deliver_input():
# Device input survives a lost HTTP waiter; cancelling its await cannot undo it.
await asyncio.sleep(0.12)
clock.now = 0.12
events.append("input_delivered")
return write_receipt
async def native(command, args, *, timeout=None):
nonlocal pending_input
calls.append((command, deepcopy(args), timeout))
if command == "inspect_control":
return missing() if args["selector"] == TARGET else observed()
pending_input = asyncio.create_task(deliver_input())
return await asyncio.wait_for(asyncio.shield(pending_input), timeout=timeout)
try:
result = await run(
"scroll_to", scroll_request(timeout_ms=50, settle_ms=50, max_scrolls=3), native
)
events.append("tool_returned")
finally:
if pending_input is not None:
await pending_input
assert events == ["input_delivered", "tool_returned"]
assert not result["success"] and result["code"] == "TIMEOUT"
assert result["execution"] == execution
assert result["scroll_results"][0]["native_receipt"] == write_receipt
assert [command for command, _, _ in calls] == [
"inspect_control",
"inspect_control",
"control_action",
]
assert calls[-1][2] == 5.0
assert not clock.sleeps
@pytest.mark.asyncio
async def test_scroll_timeout_caps_total_search_even_when_content_changes(clock):
native = ScrollNative(clock)
result = await run("scroll_to", scroll_request(timeout_ms=350, settle_ms=300), native)
assert result["code"] == "TIMEOUT" and not result["success"]
assert clock.now == pytest.approx(0.35) and native.writes <= 2
assert len(result["scroll_results"]) == native.writes
@pytest.mark.asyncio
async def test_scroll_requires_visible_target_not_merely_an_existing_scoped_node(clock):
backend = ScrollNative(clock, delayed_target=0.2)
async def native(command, args, *, timeout=None):
result = await backend(command, args, timeout=timeout)
return (
observed(text="destination", visible=False)
if result.get("code") == "NOT_FOUND"
else result
)
result = await run("scroll_to", scroll_request(), native)
assert result["success"] and backend.writes == 1
assert clock.now >= 0.4 - 1e-9
@pytest.mark.asyncio
async def test_scroll_does_not_send_next_input_while_content_is_animating(clock):
writes = []
async def native(command, args, *, timeout=None):
if command == "control_action":
writes.append(clock.now)
return completed()
if args["selector"] == TARGET:
return missing()
page = min(int(clock.now * 10 + 1e-6), 6) if writes else 0
return observed(content_signature=f"animation-{page}")
result = await run("scroll_to", scroll_request(max_scrolls=2), native)
assert result["code"] == "NO_PROGRESS" and len(writes) == 2
assert writes[1] >= 0.9 - 1e-9
@pytest.mark.asyncio
async def test_scroll_transient_visible_target_during_animation_is_not_success(clock):
writes = []
async def native(command, args, *, timeout=None):
if command == "control_action":
writes.append(clock.now)
return completed()
if args["selector"] == TARGET:
return observed() if writes and 0.1 <= clock.now < 0.2 else missing()
return observed(content_signature="after" if writes and clock.now >= 0.25 else "before")
result = await run("scroll_to", scroll_request(max_scrolls=1), native)
assert result["code"] == "MAX_SCROLLS" and not result["success"]
assert len(writes) == 1 and clock.now >= 0.55 - 1e-9
@pytest.mark.asyncio
async def test_scroll_tracks_target_outside_bounded_container_signature_sample(clock):
writes = []
async def native(command, args, *, timeout=None):
if command == "control_action":
writes.append(clock.now)
return completed()
if args["selector"] == TARGET:
return observed() if writes and 0.4 - 1e-9 <= clock.now < 0.45 - 1e-9 else missing()
return observed(content_signature="after" if writes and clock.now >= 0.1 else "before")
result = await run("scroll_to", scroll_request(max_scrolls=1), native)
assert result["code"] == "MAX_SCROLLS" and not result["success"]
assert len(writes) == 1 and clock.now >= 0.75 - 1e-9
@pytest.mark.asyncio
async def test_scroll_read_failure_after_accepted_input_preserves_receipt_without_more_input(clock):
backend = ScrollNative(clock)
async def native(command, args, *, timeout=None):
if command == "inspect_control" and backend.writes:
raise OSError("read unavailable after accepted input")
return await backend(command, args, timeout=timeout)
result = await run("scroll_to", scroll_request(), native)
assert not result["success"] and result["execution"] == "completed"
assert result["code"] == "NATIVE_ERROR" and backend.writes == 1
assert result["scroll_results"][0]["native_receipt"]["success"]
@pytest.mark.asyncio
async def test_cancelled_scroll_settling_never_sends_another_input(clock, monkeypatch):
backend = ScrollNative(clock)
async def cancelled_sleep(seconds):
raise asyncio.CancelledError
monkeypatch.setattr(native_executor, "sleep", cancelled_sleep)
with pytest.raises(asyncio.CancelledError):
await run("scroll_to", scroll_request(), backend)
assert backend.writes == 1
@pytest.mark.asyncio
@pytest.mark.parametrize("name", ["control_action", "wait_until", "scroll_to", "touch_gesture"])
async def test_cancellation_propagates_without_replaying_native_work(clock, name):
requests = {
"control_action": {"selector": SELECTOR, "action": "click"},
"wait_until": {"selector": SELECTOR, "state": "visible"},
"scroll_to": scroll_request(),
"touch_gesture": {
"frames": [{"at_ms": 0, "points": [[1, 1]]}, {"at_ms": 20, "points": [[2, 3]]}],
"target_description": "observed control",
},
}
inputs = []
async def native(command, args, *, timeout=None):
if command == "inspect_control":
if name == "wait_until":
raise asyncio.CancelledError
return missing() if args["selector"] == TARGET else observed()
inputs.append(command)
raise asyncio.CancelledError
with pytest.raises(asyncio.CancelledError):
await run(name, requests[name], native)
assert inputs == (
[]
if name == "wait_until"
else ["touch_gesture" if name == "touch_gesture" else "control_action"]
)
+270
View File
@@ -0,0 +1,270 @@
"""Mixed native sequences exercise production preflight and dispatch over mocks."""
from copy import deepcopy
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
from artemis.mcp.action_manifest import DEVICE_ACTIONS
from artemis.mcp.actuators import MockActuator
SELECTOR = {"resource_id": "fixture:id/editor"}
TOKEN = "a" * 64
def wait_step():
return {"action": "wait_until", "selector": SELECTOR, "state": "visible", "timeout_ms": 100}
def control_step():
return {
"action": "control_action",
"request": {"selector": SELECTOR, "action": "set_text", "text": " exact 😀 \n"},
}
def touch_step():
return {
"action": "touch_gesture",
"frames": [
{"at_ms": 0, "points": [[250, 500]]},
{"at_ms": 150, "points": [[250, 500]]},
{"at_ms": 400, "points": [[500, 750]]},
],
"coordinate_space": "normalized",
"target_description": "observed canvas",
}
def click_step():
return {"action": "click", "target": [50, 100], "target_description": "observed button"}
def observed():
return {
"success": True,
"execution": "observed",
"code": "OK",
"node": {
"token": TOKEN,
"content_signature": "b" * 64,
"visible": True,
"enabled": True,
"editable": True,
"password": False,
"text": " A😀B ",
"selection_start": 1,
"selection_end": 3,
"actions": ["set_text", "set_selection"],
},
}
@pytest.fixture(autouse=True)
def sequence_limit(monkeypatch):
monkeypatch.setattr(
"artemis.config.agent.load_agent_config",
lambda: SimpleNamespace(
pro=SimpleNamespace(execution=SimpleNamespace(max_burst_actions=10))
),
)
def actuator_with_native(capabilities=DEVICE_ACTIONS):
actuator = MockActuator(width=720, height=1600, capabilities=capabilities)
async def native(command, args, *, timeout=None):
if command == "inspect_control":
return observed()
return {
"success": True,
"execution": "completed",
"code": "OK",
"native_marker": command,
}
actuator.controller.native_request = AsyncMock(side_effect=native)
actuator.controller.tap_at = AsyncMock(return_value=SimpleNamespace(error=None))
actuator.controller.swipe_coords = AsyncMock()
return actuator
@pytest.mark.asyncio
@pytest.mark.parametrize(
"bad_member",
[
{**wait_step(), "timeout_ms": 0},
{**wait_step(), "state": "text"},
{**wait_step(), "timeout_ms": 3001},
{"action": "control_action", "request": {"selector": SELECTOR, "action": "set_progress"}},
{
"action": "control_action",
"request": {"selector": SELECTOR, "action": "set_text", "text": "x" * 1025},
},
{
"action": "control_action",
"request": {"selector": SELECTOR, "action": "click", "text": "unexpected"},
},
{
**touch_step(),
"frames": [
{"at_ms": 0, "points": [[100, 100]]},
{"at_ms": 400, "points": [[1001, 100]]},
],
},
{
**touch_step(),
"frames": [
{"at_ms": 0, "points": [[100, 100], [300, 100]]},
{"at_ms": 400, "points": [[300, 100], [100, 100]]},
],
},
{
**touch_step(),
"frames": [
{"at_ms": 0, "points": [[100, 100]]},
{"at_ms": 400, "points": [[200, 200], [300, 300]]},
],
},
],
)
async def test_all_native_parameters_and_paths_are_preflighted_before_even_the_first_click(
bad_member,
):
actuator = actuator_with_native()
result = await actuator.action_sequence(
[click_step(), wait_step(), control_step(), touch_step(), bad_member]
)
assert not result.ok and result.code.value == "INVALID_ARGS"
actuator.controller.native_request.assert_not_awaited()
actuator.controller.tap_at.assert_not_awaited()
actuator.controller.swipe_coords.assert_not_awaited()
assert actuator.action_history == []
@pytest.mark.asyncio
@pytest.mark.parametrize("missing_capability", ["wait_until", "control_action", "touch_gesture"])
async def test_every_native_member_capability_is_checked_before_any_input(missing_capability):
actuator = actuator_with_native(DEVICE_ACTIONS - {missing_capability})
result = await actuator.action_sequence(
[click_step(), wait_step(), control_step(), touch_step()]
)
assert not result.ok and result.code.value == "UNSUPPORTED"
assert result.metadata == {"execution": "not_started", "side_effect": "none"}
actuator.controller.native_request.assert_not_awaited()
actuator.controller.tap_at.assert_not_awaited()
actuator.controller.swipe_coords.assert_not_awaited()
@pytest.mark.asyncio
async def test_mixed_wait_control_and_touch_keep_native_receipts_and_exact_text_in_order():
actuator = actuator_with_native()
steps = [wait_step(), control_step(), touch_step()]
original = deepcopy(steps)
result = await actuator.action_sequence(steps)
assert result.ok and result.metadata["side_effect"] == "possible"
assert steps == original
calls = actuator.controller.native_request.await_args_list
assert [call.args[0] for call in calls] == [
"inspect_control",
"inspect_control",
"control_action",
"touch_gesture",
]
assert calls[2].args[1]["text"] == " exact 😀 \n"
assert calls[2].args[1]["expected_token"] == TOKEN
receipts = result.metadata["sequence_results"]
assert [item["action"] for item in receipts] == [
"wait_until",
"control_action",
"touch_gesture",
]
assert receipts[0]["read_only"] and receipts[0]["side_effect"] == "none"
assert receipts[0]["native_execution"] == "observed"
assert receipts[0]["node"]["selection_end"] == 3
assert receipts[1]["native_receipt"]["native_marker"] == "control_action"
assert receipts[2]["native_receipt"]["native_marker"] == "touch_gesture"
assert receipts[0]["ended_ms"] <= receipts[1]["started_ms"] <= receipts[2]["started_ms"]
actuator.controller.tap_at.assert_not_awaited()
actuator.controller.swipe_coords.assert_not_awaited()
@pytest.mark.asyncio
@pytest.mark.parametrize("failure", [False, True])
async def test_readonly_native_wait_never_counts_as_an_input_side_effect(failure):
actuator = actuator_with_native()
if failure:
actuator.controller.native_request.return_value = None
actuator.controller.native_request.side_effect = lambda *args, **kwargs: {
"success": False,
"execution": "not_started",
"code": "UNAVAILABLE",
"error": "No active native tree",
}
result = await actuator.action_sequence([wait_step()])
assert result.ok is not failure
assert result.metadata["side_effect"] == "none"
receipt = result.metadata["sequence_results"][0]
assert receipt["read_only"] and receipt["side_effect"] == "none"
assert receipt["native_execution"] == ("not_started" if failure else "observed")
assert [call.args[0] for call in actuator.controller.native_request.await_args_list] == [
"inspect_control"
]
@pytest.mark.asyncio
@pytest.mark.parametrize("unknown_member", ["control_action", "touch_gesture"])
async def test_unknown_native_input_stops_every_remaining_member_and_preserves_original_receipt(
unknown_member,
):
actuator = actuator_with_native()
uncertain = {
"success": False,
"execution": "unknown",
"code": "TRANSPORT_ERROR",
"error": "Reply lost; do not replay input",
"native_marker": "unacknowledged input",
}
async def native(command, args, *, timeout=None):
return observed() if command == "inspect_control" else uncertain
actuator.controller.native_request.side_effect = native
member = control_step() if unknown_member == "control_action" else touch_step()
result = await actuator.action_sequence([wait_step(), member, click_step(), touch_step()])
assert not result.ok and result.metadata["side_effect"] == "possible"
receipts = result.metadata["sequence_results"]
assert receipts[1]["execution"] == "unknown"
assert receipts[1]["native_execution"] == "unknown"
assert receipts[1]["native_receipt"] == uncertain
assert [item["execution"] for item in receipts[2:]] == ["not_started", "not_started"]
writes = [
call.args[0]
for call in actuator.controller.native_request.await_args_list
if call.args[0] != "inspect_control"
]
assert writes == [unknown_member]
actuator.controller.tap_at.assert_not_awaited()
actuator.controller.swipe_coords.assert_not_awaited()
@pytest.mark.asyncio
async def test_normalized_touch_in_sequence_is_mapped_to_physical_pixels_exactly_once():
actuator = actuator_with_native()
result = await actuator.action_sequence([touch_step()])
assert result.ok
command, sent = actuator.controller.native_request.await_args.args
assert command == "touch_gesture" and sent["screen_size"] == [720, 1600]
expected = [
{"at_ms": 0, "points": [[180, 800]]},
{"at_ms": 150, "points": [[180, 800]]},
{"at_ms": 400, "points": [[360, 1200]]},
]
assert sent["frames"] == expected
receipt = result.metadata["sequence_results"][0]
assert receipt["plan"]["frames"] == expected
assert receipt["plan"]["coordinate_space"] == "pixel"
assert receipt["args"]["coordinate_space"] == "normalized"
assert receipt["args"]["frames"] == touch_step()["frames"]
+28
View File
@@ -25,6 +25,34 @@ def delta(arguments, *, name=None, index=0):
)
@pytest.mark.parametrize(
"name", ["pinch", "touch_gesture", "read_control", "wait_until", "scroll_to"]
)
def test_native_small_actions_stop_argument_inflation(name):
guard = ToolArgumentGuard()
guard.observe(
SimpleNamespace(
tool_call_chunks=[{"index": 0, "name": name, "args": "x" * SMALL_ACTION_ARGUMENT_LIMIT}]
)
)
with pytest.raises(ToolArgumentOverflow) as error:
guard.observe(SimpleNamespace(tool_call_chunks=[{"index": 0, "args": "x"}]))
assert error.value.details["tool"] == name
assert error.value.details["limit_chars"] == SMALL_ACTION_ARGUMENT_LIMIT
def test_standalone_native_text_allows_valid_escaped_text_but_remains_bounded():
guard = ToolArgumentGuard()
escaped = json.dumps({"text": "中" * 65536})
assert len(escaped) > SMALL_ACTION_ARGUMENT_LIMIT
guard.observe(
SimpleNamespace(tool_call_chunks=[{"index": 0, "name": "control_action", "args": escaped}])
)
with pytest.raises(ToolArgumentOverflow) as error:
guard.observe(SimpleNamespace(tool_call_chunks=[{"index": 0, "args": "x" * (512 * 1024)}]))
assert error.value.details["limit_chars"] == 512 * 1024
@pytest.fixture(autouse=True)
def isolate(monkeypatch, tmp_path):
llm._ENDPOINT_BREAKER._states.clear()
@@ -0,0 +1,209 @@
"""Hierarchy native semantics must survive both supported observation encodings."""
from copy import deepcopy
import json
import xml.etree.ElementTree as ET
import pytest
from artemis.clients.accessibility_client import normalize_helper_elements
from artemis.clients.ui_automator_client import _parse_hierarchy_xml_to_elements
from artemis.utils.native_control_state import informative_native_control, native_control_state
from artemis.utils.visualization import format_minimal_list_with_elements
RANGE = {"min": 0, "max": 100, "current": 37.5, "type": 1}
ACTIONS = ["set_progress", "set_selection", "set_text"]
def hierarchy_node(**overrides):
return {
"text": "",
"content-desc": "",
"class": "android.widget.SeekBar",
"package": "com.artemis.fixture",
"resource-id": "com.artemis.fixture:id/range",
"bounds": "[10,20][210,80]",
"enabled": True,
"editable": False,
"password": False,
"checked": False,
"checkable": False,
"selected": False,
"focused": False,
"scrollable": False,
"visible-to-user": True,
"selection_start": -1,
"selection_end": -1,
"range": deepcopy(RANGE),
"actions": list(ACTIONS),
**overrides,
}
def encode_node(node, encoding):
if encoding == "json":
return normalize_helper_elements(json.loads(json.dumps([node])))[0]
attributes = {}
for key, value in node.items():
name = key.replace("_", "-") if key in {"selection_start", "selection_end"} else key
attributes[name] = (
json.dumps(value, ensure_ascii=False)
if isinstance(value, (dict, list, bool)) or value is None
else str(value)
)
root = ET.Element("hierarchy")
ET.SubElement(root, "node", attributes)
return _parse_hierarchy_xml_to_elements(ET.tostring(root, encoding="unicode"))[0]
@pytest.mark.parametrize("encoding", ["xml", "json"])
def test_native_range_actions_flags_and_utf16_selection_map_equally_from_xml_and_json(encoding):
state = native_control_state(
encode_node(hierarchy_node(editable=True, selection_start=1, selection_end=3), encoding)
)
assert state == {
"range": RANGE,
"actions": ACTIONS,
"enabled": True,
"scrollable": False,
"editable": True,
"password": False,
"checked": False,
"checkable": False,
"selected": False,
"focused": False,
"selection_start": 1,
"selection_end": 3,
}
@pytest.mark.parametrize("encoding", ["xml", "json"])
@pytest.mark.parametrize(
"semantics",
[
{"range": RANGE, "actions": []},
{"range": None, "actions": ["set_progress"]},
{"range": None, "actions": ["click"]},
{"range": None, "actions": [], "scrollable": True},
],
)
def test_unnamed_native_controls_are_discoverable_in_the_actual_visualization(encoding, semantics):
node = encode_node(hierarchy_node(**semantics), encoding)
text, elements, labels = format_minimal_list_with_elements([node], 400, 800)
assert len(elements) == 1 and labels == ["1"]
element = elements[0]
assert element["text"] == "(native control)"
assert element["native_text"] == "" and element["content_desc"] == ""
assert element["resource_id"] == "com.artemis.fixture:id/range"
assert element["package_name"] == "com.artemis.fixture"
assert element["class"] == "android.widget.SeekBar"
assert element["center"] == [110, 50] and element["bounds"] == [10, 20, 210, 80]
assert "Control: '(native control)'" in text
assert 'package_name: "com.artemis.fixture"' in text
if semantics["range"] is not None:
assert element["range"] == RANGE and '"current": 37.5' in text
if semantics["actions"]:
assert element["actions"] == semantics["actions"]
assert "actions: " + ",".join(semantics["actions"]) in text
@pytest.mark.parametrize("encoding", ["xml", "json"])
def test_native_text_description_package_and_selection_preserve_exact_observed_data(encoding):
raw = hierarchy_node(
text=" A😀B \n\t",
**{
"content-desc": " description \n\t",
"class": "android.widget.EditText",
"package": "com.artemis.fixture.editor",
"editable": True,
"selection_start": 1,
"selection_end": 3,
},
)
node = encode_node(raw, encoding)
original = deepcopy(node)
text, elements, _ = format_minimal_list_with_elements([node], 400, 800)
element = elements[0]
assert element["native_text"] == raw["text"]
assert element["input_value"] == raw["text"]
assert element["content_desc"] == raw["content-desc"]
assert element["package_name"] == raw["package"]
assert (element["selection_start"], element["selection_end"]) == (1, 3)
assert element["range"] == RANGE and element["actions"] == ACTIONS
assert json.dumps(raw["content-desc"], ensure_ascii=False) in text
assert node == original
@pytest.mark.parametrize(
"bad_range",
[
None,
[],
"invalid json",
"{}",
{"min": 0, "max": 100},
{"min": 10, "max": 0, "current": 5},
{"min": 0, "max": 10, "current": 11},
{"min": False, "max": 10, "current": 1},
{"min": "0", "max": 10, "current": 1},
{"min": 0, "max": float("inf"), "current": 1},
{"min": 0, "max": 10, "current": float("nan")},
],
)
def test_malformed_range_does_not_hide_other_valid_native_metadata(bad_range):
assert native_control_state({"range": bad_range, "actions": ["click"], "enabled": "true"}) == {
"actions": ["click"],
"enabled": True,
}
@pytest.mark.parametrize("encoding", ["xml", "json"])
@pytest.mark.parametrize("field", ["min", "max", "current"])
def test_huge_range_integers_are_omitted_without_crashing_observation(encoding, field):
value = {**RANGE, field: 10**1000}
node = encode_node(hierarchy_node(range=value), encoding)
state = native_control_state(node)
assert "range" not in state and state["actions"] == ACTIONS
_, elements, _ = format_minimal_list_with_elements([node], 400, 800)
assert len(elements) == 1 and "range" not in elements[0]
@pytest.mark.parametrize(
"key", ["selection_start", "selection-start", "selection_end", "selection-end"]
)
@pytest.mark.parametrize("bad_offset", [True, 1.5, "1.5", "--1", "²", -2, 10**1000, "9" * 100])
def test_invalid_or_impossible_selection_offsets_are_ignored_without_raising(key, bad_offset):
state = native_control_state({key: bad_offset, "enabled": True})
assert state == {"enabled": True}
@pytest.mark.parametrize("value", [-1, 0, 3, 2**31 - 1, "-1", "0", "3", str(2**31 - 1)])
def test_supported_native_int_selection_offsets_are_kept(value):
assert native_control_state({"selection-start": value}) == {"selection_start": int(value)}
def test_encoded_metadata_is_bounded_and_rejects_nonstring_action_names():
state = native_control_state(
{
"range": " " * 8193,
"actions": ["click", None, 7, False, "x" * 65, *["set_progress"] * 40],
}
)
assert "range" not in state
assert state["actions"] == ["click", *["set_progress"] * 27]
assert "actions" not in native_control_state({"actions": " " * 8193})
@pytest.mark.parametrize("value", [None, 0, 1, "TRUE", "False", "yes", {}, []])
def test_metadata_flags_require_actual_boolean_values_or_canonical_xml_strings(value):
assert native_control_state({"enabled": value}) == {}
def test_passive_empty_layouts_do_not_become_actionable_controls():
state = native_control_state({"actions": ["focus", "show_on_screen"], "range": None})
assert not informative_native_control(state)
text, elements, labels = format_minimal_list_with_elements(
[hierarchy_node(actions=[], range=None)], 400, 800
)
assert (text, elements, labels) == ("", [], [])