feat(android): add native controls and continuous multi-touch
This commit is contained in:
1 parent
3dd96078db
commit
4b12ae7728
70 files changed
+13105
-179
No files matched your search
@@ -0,0 +1,103 @@
|
||||
"""Guard the prompt evidence contract; no model or device is invoked here."""
|
||||
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.agents.checker.checker import (
|
||||
CheckReport,
|
||||
assemble_checker_prompt_segments,
|
||||
run_final_check,
|
||||
)
|
||||
from artemis.context import ArtemisContext, ExecutionSetup
|
||||
|
||||
|
||||
@pytest.mark.parametrize("entry", ["checkpoint", "final"])
|
||||
def test_bound_persistent_readout_can_supply_precision_beyond_pixel_ticks(entry):
|
||||
prompt = assemble_checker_prompt_segments(entry, [], probe_tool_registered=False)
|
||||
|
||||
# Native origin alone is insufficient: the rule must bind the displayed value
|
||||
# to the selected state and require stability plus the visible pointer check.
|
||||
for requirement in (
|
||||
"标签、节点角色/层级",
|
||||
"目标控件当前所选值或固定指针所选时间",
|
||||
"不能仅因原生节点含有数值或执行者称其可信",
|
||||
"在被验收状态中持续显示",
|
||||
"引用已停稳或暂停的证据",
|
||||
"与同一控件的指针及可辨刻度一致",
|
||||
"结合读数精度与用户容差",
|
||||
"粗刻度无法独立区分秒数,不等于",
|
||||
"无需再由粗像素刻度独立插值证明同一精确值",
|
||||
):
|
||||
assert requirement in prompt
|
||||
|
||||
assert "拖动浮层或原生时间读数与固定指针刻度冲突时" not in prompt
|
||||
|
||||
|
||||
@pytest.mark.parametrize("entry", ["checkpoint", "final"])
|
||||
def test_native_origin_does_not_promote_unbound_or_conflicting_evidence(entry):
|
||||
prompt = assemble_checker_prompt_segments(entry, [], probe_tool_registered=False)
|
||||
|
||||
for excluded_source in ("事件卡片时间", "视频画面水印", "录制起止时间", "拖动/松手浮层"):
|
||||
assert excluded_source in prompt
|
||||
assert "即使它们也能通过原生节点读取,仍须先证明上述绑定与持久性" in prompt
|
||||
assert "可辨精度上存在实际冲突" in prompt
|
||||
assert "读数仍在变化、绑定不明或无法确认已停稳/暂停" in prompt
|
||||
assert "应判证据不足(inconclusive)" in prompt
|
||||
assert "不得挑选有利读数或忽略矛盾" in prompt
|
||||
assert "可信读数明确超出用户容差时仍按通用规则判定" in prompt
|
||||
|
||||
|
||||
@pytest.mark.parametrize("entry", ["checkpoint", "final"])
|
||||
def test_pixel_fallback_remains_quantitative_and_requires_post_action_evidence(entry):
|
||||
prompt = assemble_checker_prompt_segments(entry, [], probe_tool_registered=False)
|
||||
fallback = prompt.split("没有可信持久读数时,", 1)[1]
|
||||
|
||||
assert "动作后已停稳截图的两处真实数值刻度与固定指针位置作线性插值" in fallback
|
||||
assert "单位换算、估读不确定性及目标误差" in fallback
|
||||
assert "目标误差加估读不确定性不超过用户容差" in fallback
|
||||
assert "刻度非线性或无法确定已停稳,仍应判证据不足" in fallback
|
||||
assert "操作前的校准计划及回执误差不能替代动作后测量" in fallback
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_final_review_receives_live_readout_and_rules_separately_from_report_claims():
|
||||
ctx = MagicMock(spec=ArtemisContext)
|
||||
ctx.execution_setup = ExecutionSetup()
|
||||
ctx.data_engine = MagicMock()
|
||||
ctx.data_engine.base_dir = "unused"
|
||||
ctx.data_engine.get_agent_friendly_steps.return_value = []
|
||||
ctx.data_engine.get_step_number.return_value = 7
|
||||
elements = "[1] 当前所选值: 23.5; [2] 已暂停"
|
||||
review = AsyncMock(return_value=CheckReport(verdicts=[]))
|
||||
|
||||
with (
|
||||
patch("artemis.agents.checker.checker._run_check_loop", new=review),
|
||||
patch(
|
||||
"artemis.agents.checker.checker._capture_final_screen",
|
||||
new=AsyncMock(return_value=("captured-image", elements)),
|
||||
),
|
||||
):
|
||||
await run_final_check(
|
||||
ctx,
|
||||
goal="核对当前所选值及容差",
|
||||
plan_text="",
|
||||
ledger=[],
|
||||
check_items=[],
|
||||
execution_report="执行者声称当前所选值为 88",
|
||||
)
|
||||
|
||||
review.assert_awaited_once()
|
||||
messages = review.await_args.args[1]
|
||||
assert "无需再由粗像素刻度独立插值证明同一精确值" in messages[0].content
|
||||
assert "不得挑选有利读数或忽略矛盾" in messages[0].content
|
||||
content = messages[1].content
|
||||
text = content[0]["text"]
|
||||
assert "Executor's Final Report (claims to verify, not independent evidence)" in text
|
||||
assert "执行者声称当前所选值为 88" in text
|
||||
assert f"# Final Screen Elements\n{elements}" in text
|
||||
assert any(
|
||||
block.get("type") == "image_url"
|
||||
and block["image_url"]["url"] == "data:image/jpeg;base64,captured-image"
|
||||
for block in content
|
||||
)
|
||||
@@ -0,0 +1,194 @@
|
||||
"""Actual action-result/history/transcript rendering must keep native evidence."""
|
||||
|
||||
from copy import deepcopy
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
|
||||
import pytest
|
||||
|
||||
from artemis.agents.validator.action_execution import exec_action
|
||||
from artemis.mcp.action_types import ActionCode, ActionResult
|
||||
from artemis.memory.transcript import EXECUTION_RESULT_MARKER, TranscriptLedger
|
||||
from artemis.utils.task_tree import format_result_clean
|
||||
|
||||
|
||||
NATIVE_ACTIONS = (
|
||||
"pinch",
|
||||
"touch_gesture",
|
||||
"read_control",
|
||||
"control_action",
|
||||
"wait_until",
|
||||
"scroll_to",
|
||||
)
|
||||
TOKEN = "a" * 64
|
||||
SELECTOR = {"resource_id": "fixture:id/editor"}
|
||||
|
||||
|
||||
def action_arguments(action):
|
||||
drag = {
|
||||
"start": [191, 501],
|
||||
"end": [192, 532],
|
||||
"coordinate_space": "pixel",
|
||||
"target_description": "observed canvas",
|
||||
}
|
||||
return {
|
||||
"pinch": {
|
||||
"center": [200, 400],
|
||||
"start_span": 100,
|
||||
"end_span": 200,
|
||||
"coordinate_space": "pixel",
|
||||
"target_description": "observed canvas",
|
||||
},
|
||||
"touch_gesture": {
|
||||
"frames": [
|
||||
{"at_ms": 0, "points": [[191, 501]]},
|
||||
{"at_ms": 400, "points": [[192, 532]]},
|
||||
],
|
||||
"coordinate_space": "pixel",
|
||||
"target_description": "observed canvas",
|
||||
},
|
||||
"read_control": {"selector": SELECTOR},
|
||||
"control_action": {
|
||||
"selector": SELECTOR,
|
||||
"action": "set_selection",
|
||||
"selection_start": 3,
|
||||
"selection_end": 5,
|
||||
},
|
||||
"wait_until": {"selector": SELECTOR, "state": "value", "expected_value": 37.5},
|
||||
"scroll_to": {"container": {"resource_id": "fixture:id/list"}, "target": SELECTOR},
|
||||
"drag": drag,
|
||||
"action_sequence": {"steps": [{"action": "drag", **drag}]},
|
||||
}[action]
|
||||
|
||||
|
||||
def control_observation():
|
||||
return {
|
||||
"resource_id": "fixture:id/editor",
|
||||
"text": " A😀B \n",
|
||||
"selection_start": 3,
|
||||
"selection_end": 5,
|
||||
"range": {"min": 0, "max": 100, "current": 37.5, "type": 1},
|
||||
"token": TOKEN,
|
||||
"content_signature": "b" * 64,
|
||||
"editable": True,
|
||||
"password": False,
|
||||
}
|
||||
|
||||
|
||||
def decode_receipt(text, label):
|
||||
marker = f"{label} receipt: "
|
||||
assert marker in text
|
||||
return json.loads(text.split(marker, 1)[1])
|
||||
|
||||
|
||||
async def render_result_pipeline(action, metadata, failed):
|
||||
message = (
|
||||
"Native reply is uncertain; observe before continuing." if failed else "Input accepted."
|
||||
)
|
||||
result = (
|
||||
ActionResult.failure(action, message, code=ActionCode.TIMEOUT, metadata=metadata)
|
||||
if failed
|
||||
else ActionResult.success(action, message, metadata=metadata)
|
||||
)
|
||||
session = SimpleNamespace(call=AsyncMock(return_value=result))
|
||||
context = SimpleNamespace(
|
||||
data_engine=None, device=SimpleNamespace(device_width=400, device_height=800)
|
||||
)
|
||||
args = action_arguments(action)
|
||||
item = {"action": action, "args": args}
|
||||
ok, error = await exec_action(context, session, item)
|
||||
assert ok is not failed
|
||||
session.call.assert_awaited_once_with(action, args)
|
||||
assert item["result"] == result.agent_message
|
||||
report = {
|
||||
"status": "failed" if failed else "dispatched",
|
||||
"execution": [{**item, "attempts": [f"Error: {error}" if failed else "Dispatched"]}],
|
||||
}
|
||||
history = format_result_clean(report)
|
||||
assert isinstance(history, str)
|
||||
|
||||
ledger = TranscriptLedger(clock=lambda: 0.0)
|
||||
ledger.stage_turn(
|
||||
[
|
||||
HumanMessage(content="Observed native controls"),
|
||||
AIMessage(content="", tool_calls=[{"name": action, "args": {}, "id": "native-call"}]),
|
||||
ToolMessage(content="Action Recorded", tool_call_id="native-call"),
|
||||
]
|
||||
)
|
||||
ledger.commit_staged(step_key="native-step", validator_result=report)
|
||||
rendered = ledger.render([HumanMessage(content="Next observation")])
|
||||
execution_results = [
|
||||
message.content[0]["text"]
|
||||
for message in rendered
|
||||
if isinstance(message, HumanMessage)
|
||||
and isinstance(message.content, list)
|
||||
and message.content
|
||||
and EXECUTION_RESULT_MARKER in message.content[0].get("text", "")
|
||||
]
|
||||
assert len(execution_results) == 1
|
||||
transcript = execution_results[0]
|
||||
assert f"Status: {report['status']}" in transcript
|
||||
if failed:
|
||||
assert "Error:" in history and "Error:" in transcript
|
||||
return result.agent_message, history, transcript
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("action", NATIVE_ACTIONS)
|
||||
@pytest.mark.parametrize("failed", [False, True])
|
||||
async def test_every_native_action_keeps_readings_selection_range_and_token_in_actual_result_rendering(
|
||||
action, failed
|
||||
):
|
||||
node = control_observation()
|
||||
read_only = action in {"read_control", "wait_until"}
|
||||
execution = (
|
||||
("not_started" if read_only else "unknown")
|
||||
if failed
|
||||
else ("observed" if read_only else "completed")
|
||||
)
|
||||
# The renderer must retain optional observed evidence even on gesture receipts.
|
||||
metadata = {
|
||||
"action": action,
|
||||
"execution": execution,
|
||||
"node": deepcopy(node),
|
||||
"native_receipt": {
|
||||
"success": not failed,
|
||||
"execution": execution,
|
||||
"code": "TIMEOUT" if failed else "OK",
|
||||
"node": deepcopy(node),
|
||||
},
|
||||
"verification": "pending_observation",
|
||||
}
|
||||
stages = await render_result_pipeline(action, metadata, failed)
|
||||
for rendered in stages:
|
||||
recovered = decode_receipt(rendered, "Native")
|
||||
assert recovered == metadata
|
||||
assert recovered["node"]["text"] == " A😀B \n"
|
||||
assert recovered["node"]["range"]["current"] == 37.5
|
||||
assert (recovered["node"]["selection_start"], recovered["node"]["selection_end"]) == (3, 5)
|
||||
assert recovered["node"]["token"] == TOKEN
|
||||
assert recovered["native_receipt"]["success"] is not failed
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("action", ["drag", "action_sequence"])
|
||||
@pytest.mark.parametrize("failed", [False, True])
|
||||
async def test_existing_drag_and_sequence_geometry_and_partial_receipts_still_render(
|
||||
action, failed
|
||||
):
|
||||
metadata = {
|
||||
"execution": "partial" if failed else "completed",
|
||||
"coordinate_space": "pixel",
|
||||
"start": [191, 501],
|
||||
"end": [192, 532],
|
||||
"calibration": {"axis": "y", "target_value": 37.5},
|
||||
}
|
||||
if action == "action_sequence":
|
||||
metadata["sequence_results"] = [
|
||||
{"action": "click", "execution": "completed", "completed_taps": 1},
|
||||
{"action": "drag", "execution": "unknown" if failed else "completed"},
|
||||
]
|
||||
for rendered in await render_result_pipeline(action, metadata, failed):
|
||||
assert decode_receipt(rendered, "Gesture") == metadata
|
||||
@@ -35,6 +35,7 @@ from langchain_core.messages import AIMessage, HumanMessage, SystemMessage, Tool
|
||||
from artemis.agents.operator.operator import OperatorNode
|
||||
from artemis.agents.operator.prompts import load_operator_prompts
|
||||
from artemis.agents.operator.prompts import (
|
||||
OPERATOR_PROMPT_TOOLSET,
|
||||
PLAN_HISTORY_STATIC_POINTER,
|
||||
PLAN_HISTORY_TEMPLATE_SECTION,
|
||||
PromptBuilder,
|
||||
@@ -54,8 +55,8 @@ from artemis.memory.transcript import (
|
||||
|
||||
# SHA-256 snapshots of the legacy system message with the fixed inputs below.
|
||||
# Update these when an intentional template change alters the rendered prompt.
|
||||
GOLDEN_EMPTY_PLAN = "23c3d6b5fc3694275e7015f7bd0901bde54aae54fec7a8c68d4f8529c65ef4bc"
|
||||
GOLDEN_SENTINEL_PLAN = "370c53f1d088a4120c63fbaea6d895f121c683b8d3c8cc7d61105d6e1d223128"
|
||||
GOLDEN_EMPTY_PLAN = "f5ee50f4923fed4cb943cb80087d39ffffe66f259cfae4a54f27a244d7b06453"
|
||||
GOLDEN_SENTINEL_PLAN = "bb725e2d096f6aa0cf1773809b2d0e672475589782b9c72348d9eaf8481230b9"
|
||||
|
||||
SCREENSHOT_B64 = base64.b64encode(b"fake-jpeg-bytes").decode("utf-8")
|
||||
|
||||
@@ -84,6 +85,42 @@ async def test_legacy_prompt_bytes_match_chinese_prompt_golden():
|
||||
assert hashlib.sha256(sentinel.encode("utf-8")).hexdigest() == GOLDEN_SENTINEL_PLAN
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("transcript", [False, True])
|
||||
async def test_legacy_and_transcript_prompts_omit_all_unavailable_native_capabilities(transcript):
|
||||
native_tools = {
|
||||
"pinch",
|
||||
"touch_gesture",
|
||||
"read_control",
|
||||
"control_action",
|
||||
"wait_until",
|
||||
"scroll_to",
|
||||
}
|
||||
available_tools = OPERATOR_PROMPT_TOOLSET - native_tools
|
||||
ctx = SimpleNamespace(
|
||||
execution_setup=None,
|
||||
actuator=SimpleNamespace(capabilities=lambda: available_tools),
|
||||
data_engine=None,
|
||||
)
|
||||
state = SimpleNamespace(initial_goal="FILTERED_FIXED_GOAL")
|
||||
if transcript:
|
||||
system = render_transcript_static_system(load_operator_prompts(), ctx, state)
|
||||
else:
|
||||
builder = PromptBuilder()
|
||||
await TemplatePromptComponent()(
|
||||
builder,
|
||||
state,
|
||||
ctx,
|
||||
prompts=load_operator_prompts(),
|
||||
template_name="main_template",
|
||||
)
|
||||
system = "".join(builder.system_parts)
|
||||
assert "FILTERED_FIXED_GOAL" in system
|
||||
assert "`click`" in system and "`drag`" in system
|
||||
for tool in native_tools:
|
||||
assert tool not in system
|
||||
|
||||
|
||||
def test_main_template_contains_exactly_one_plan_history_section():
|
||||
template = load_operator_prompts()["main_template"]
|
||||
assert template.count(PLAN_HISTORY_TEMPLATE_SECTION) == 1
|
||||
|
||||
@@ -38,6 +38,14 @@ OPERATOR_DEVICE_TOOLS = (
|
||||
"wait_for_delay",
|
||||
"long_press",
|
||||
)
|
||||
OPERATOR_NATIVE_TOOLS = (
|
||||
"pinch",
|
||||
"touch_gesture",
|
||||
"read_control",
|
||||
"control_action",
|
||||
"wait_until",
|
||||
"scroll_to",
|
||||
)
|
||||
|
||||
|
||||
def _tool_refs(text: str, tool: str) -> list[str]:
|
||||
@@ -88,6 +96,7 @@ def test_full_toolset_is_the_default():
|
||||
@pytest.mark.parametrize(
|
||||
"tool",
|
||||
OPERATOR_DEVICE_TOOLS
|
||||
+ OPERATOR_NATIVE_TOOLS
|
||||
+ ("video_analyzer", "search_history", "replay_steps", "get_step_screenshot"),
|
||||
)
|
||||
def test_removed_tool_leaves_no_reference(tool):
|
||||
@@ -99,6 +108,23 @@ def test_removed_tool_leaves_no_reference(tool):
|
||||
assert not _tool_refs(out, tool), f"'{tool}' still referenced in {name}"
|
||||
|
||||
|
||||
def test_removing_all_native_capabilities_leaves_no_native_names_in_any_template():
|
||||
available_tools = OPERATOR_PROMPT_TOOLSET - set(OPERATOR_NATIVE_TOOLS)
|
||||
for name, template in load_operator_prompts().items():
|
||||
out = apply_operator_prompt_contract(template, available_tools=available_tools)
|
||||
for tool in OPERATOR_NATIVE_TOOLS:
|
||||
assert tool not in out, f"'{tool}' still advertised without native capability in {name}"
|
||||
assert "`click`" in out and "`drag`" in out
|
||||
assert (
|
||||
"Physical device actions (`click`, `action_sequence`, `drag`, `input_text`, `swipe`,"
|
||||
" `press_key`, `manage_app`, `wait_for_delay`, `long_press`)" in out
|
||||
)
|
||||
assert (
|
||||
"Turn-Ending Action (`click`, `action_sequence`, `drag`, `swipe`, `input_text`,"
|
||||
" `long_press`, `press_key`, `manage_app`, or `wait_for_delay`)" in out
|
||||
)
|
||||
|
||||
|
||||
def test_removed_swipe_takes_its_teaching_along():
|
||||
"""Gating removes the drag/slider teaching, not just the backticked name."""
|
||||
template = load_operator_prompts()["main_template"]
|
||||
@@ -131,26 +157,30 @@ def test_reduced_enums_stay_well_formed():
|
||||
available_tools=OPERATOR_PROMPT_TOOLSET - {"manage_app", "wait_for_delay"},
|
||||
)
|
||||
assert (
|
||||
"Physical device actions (`click`, `action_sequence`, `drag`, `input_text`, `swipe`, `press_key`, `long_press`)"
|
||||
in out
|
||||
"Physical device actions (`click`, `action_sequence`, `drag`, `pinch`, `touch_gesture`,"
|
||||
" `read_control`, `control_action`, `wait_until`, `scroll_to`, `input_text`, `swipe`,"
|
||||
" `press_key`, `long_press`)" in out
|
||||
)
|
||||
assert (
|
||||
"Turn-Ending Action (`click`, `action_sequence`, `drag`, `swipe`, `input_text`, `long_press`, or `press_key`)"
|
||||
in out
|
||||
"Turn-Ending Action (`click`, `action_sequence`, `drag`, `pinch`, `touch_gesture`,"
|
||||
" `read_control`, `control_action`, `wait_until`, `scroll_to`, `swipe`, `input_text`,"
|
||||
" `long_press`, or `press_key`)" in out
|
||||
)
|
||||
|
||||
|
||||
def test_full_set_enum_slots_render_verbatim():
|
||||
"""With every tool present the historical enumeration wording is reproduced."""
|
||||
"""Every supported tool appears in the exact physical and turn-ending order."""
|
||||
template = load_operator_prompts()["main_template"]
|
||||
out = apply_operator_prompt_contract(template)
|
||||
assert (
|
||||
"Physical device actions (`click`, `action_sequence`, `drag`, `input_text`, `swipe`, `press_key`,"
|
||||
" `manage_app`, `wait_for_delay`, `long_press`)" in out
|
||||
"Physical device actions (`click`, `action_sequence`, `drag`, `pinch`, `touch_gesture`,"
|
||||
" `read_control`, `control_action`, `wait_until`, `scroll_to`, `input_text`, `swipe`,"
|
||||
" `press_key`, `manage_app`, `wait_for_delay`, `long_press`)" in out
|
||||
)
|
||||
assert (
|
||||
"Turn-Ending Action (`click`, `action_sequence`, `drag`, `swipe`, `input_text`, `long_press`,"
|
||||
" `press_key`, `manage_app`, or `wait_for_delay`)" in out
|
||||
"Turn-Ending Action (`click`, `action_sequence`, `drag`, `pinch`, `touch_gesture`,"
|
||||
" `read_control`, `control_action`, `wait_until`, `scroll_to`, `swipe`, `input_text`,"
|
||||
" `long_press`, `press_key`, `manage_app`, or `wait_for_delay`)" in out
|
||||
)
|
||||
assert "Helper/Subagent tools (`ask_explorer`, `ask_diagnoser`, `video_analyzer`)" in out
|
||||
assert "ADB/task tools (`run_adb_command`, `manage_task`, `android_file`)" in out
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
"""Native input is one-shot and remains owned through uncertain transport cleanup."""
|
||||
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
import urllib.error
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.clients import accessibility_client as module
|
||||
from artemis.clients.accessibility_client import AccessibilityClient
|
||||
from artemis.clients.screen_client_factory import FallbackScreenClient
|
||||
from artemis.runtime.helper_manager import HelperSession
|
||||
from artemis.runtime.native_protocol import input_completion_seconds
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def native(monkeypatch):
|
||||
manager = Mock()
|
||||
client = AccessibilityClient("test-device", manager=manager)
|
||||
client._session = HelperSession(
|
||||
serial="test-device",
|
||||
local_port=40001,
|
||||
transport_id="1",
|
||||
version_code=12,
|
||||
version_name="1.3.0",
|
||||
owns_forward=True,
|
||||
token="test-only",
|
||||
protocol_version=3,
|
||||
capabilities=("inspect_control", "control_action", "touch_gesture", "cancel_gesture"),
|
||||
)
|
||||
clock = [0.0]
|
||||
sleeps = []
|
||||
|
||||
def sleep(seconds):
|
||||
sleeps.append(seconds)
|
||||
clock[0] += seconds
|
||||
|
||||
monkeypatch.setattr(module, "time", SimpleNamespace(monotonic=lambda: clock[0], sleep=sleep))
|
||||
client._http_once = Mock(return_value=b'{"success":true,"execution":"completed","code":"OK"}')
|
||||
return client, manager, clock, sleeps
|
||||
|
||||
|
||||
def test_exactly_one_request_and_command_cannot_be_overridden(native):
|
||||
client, manager, _, sleeps = native
|
||||
receipt = {
|
||||
"success": True,
|
||||
"execution": "observed",
|
||||
"code": "OK",
|
||||
"node": {"selection_start": 3},
|
||||
}
|
||||
client._http_once.return_value = json.dumps(receipt).encode()
|
||||
assert client.native_request("inspect_control", {"cmd": "swipe"}, timeout=0.02) == receipt
|
||||
client._http_once.assert_called_once_with(
|
||||
client.session, "/action", {"cmd": "inspect_control"}, 0.02
|
||||
)
|
||||
assert sleeps == []
|
||||
manager.attach.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"status,execution,attempts,repairs",
|
||||
[
|
||||
(400, "not_started", 1, 0),
|
||||
(401, "not_started", 1, 0),
|
||||
(403, "not_started", 1, 0),
|
||||
(503, "unknown", 1, 0),
|
||||
],
|
||||
)
|
||||
def test_http_rejection_never_replays_native_input(native, status, execution, attempts, repairs):
|
||||
client, manager, _, sleeps = native
|
||||
client._http_once.side_effect = urllib.error.HTTPError(
|
||||
"http://test", status, "rejected", {}, None
|
||||
)
|
||||
result = client.native_request("control_action", {"action": "click"}, timeout=0.02)
|
||||
assert result["execution"] == execution and not result["success"]
|
||||
assert client._http_once.call_count == attempts
|
||||
assert client._http_once.call_args.args[-1] >= 5
|
||||
assert manager.push_token.call_count == repairs
|
||||
manager.reattach.assert_not_called()
|
||||
assert bool(sleeps) == (execution == "unknown")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"response", [b"broken", b"[]", b"{}", b'{"success":true,"execution":"unknown"}']
|
||||
)
|
||||
def test_uncertain_write_drains_without_retry(native, response):
|
||||
client, _, clock, sleeps = native
|
||||
client._http_once.return_value = response
|
||||
result = client.native_request("control_action", {"action": "click"})
|
||||
assert result["execution"] == "unknown"
|
||||
assert sleeps == [5.0] and clock[0] == 5.0
|
||||
client._http_once.assert_called_once()
|
||||
|
||||
|
||||
def test_partial_transport_elapsed_time_counts_toward_input_drain(native):
|
||||
client, _, clock, sleeps = native
|
||||
|
||||
def broken(*args):
|
||||
clock[0] = 2
|
||||
raise OSError("response lost after dispatch")
|
||||
|
||||
client._http_once.side_effect = broken
|
||||
assert client.native_request("control_action", {})["execution"] == "unknown"
|
||||
assert sleeps == [3.0] and clock[0] == 5
|
||||
client._http_once.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("command", ["inspect_control", "cancel_gesture"])
|
||||
def test_non_input_calls_do_not_drain_on_failure(native, command):
|
||||
client, _, _, sleeps = native
|
||||
client._http_once.side_effect = OSError("offline")
|
||||
result = client.native_request(command, {}, timeout=0.02)
|
||||
assert not result["success"] and sleeps == []
|
||||
client._http_once.assert_called_once()
|
||||
|
||||
|
||||
def test_no_session_or_capability_cannot_trigger_attach_or_input(native):
|
||||
client, manager, _, sleeps = native
|
||||
client._session.capabilities = ()
|
||||
assert client.native_request("touch_gesture", {})["code"] == "UNSUPPORTED"
|
||||
client._session = None
|
||||
assert client.native_capabilities == frozenset()
|
||||
assert client.native_request("control_action", {})["execution"] == "not_started"
|
||||
client._http_once.assert_not_called()
|
||||
manager.attach.assert_not_called()
|
||||
assert sleeps == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"duration,budget", [(0, 5), (1000, 6), (10000, 15), (999999, 15), (None, 15)]
|
||||
)
|
||||
def test_touch_completion_budget_is_bounded(duration, budget):
|
||||
assert input_completion_seconds("touch_gesture", {"frames": [{"at_ms": duration}]}) == budget
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend", ["helper", "uiautomator"])
|
||||
def test_native_input_never_uses_fallback_dispatch(backend):
|
||||
client = object.__new__(FallbackScreenClient)
|
||||
client._active_backend = backend
|
||||
client._helper = Mock()
|
||||
client._helper.native_capabilities = frozenset({"control_action"})
|
||||
unknown = {"success": False, "execution": "unknown", "code": "TRANSPORT_ERROR"}
|
||||
client._helper.native_request.return_value = unknown
|
||||
client._call = Mock(side_effect=AssertionError("fallback is forbidden"))
|
||||
result = client.native_request("control_action", {})
|
||||
if backend == "helper":
|
||||
assert result is unknown and client.native_capabilities == frozenset({"control_action"})
|
||||
client._helper.native_request.assert_called_once()
|
||||
else:
|
||||
assert result["execution"] == "not_started" and client.native_capabilities == frozenset()
|
||||
client._helper.native_request.assert_not_called()
|
||||
client._call.assert_not_called()
|
||||
@@ -0,0 +1,70 @@
|
||||
"""Cancellation drains input threads, while read-only work cannot hold the deadline."""
|
||||
|
||||
import asyncio
|
||||
from threading import Event
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.drivers.android.adb_driver import AndroidAdbDriver
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("command", ["control_action", "touch_gesture", "inspect_control"])
|
||||
async def test_cancel_keeps_input_owned_until_transport_finishes(command):
|
||||
started, release, finished, cancel_seen = Event(), Event(), Event(), Event()
|
||||
|
||||
def request(name, args, *, timeout):
|
||||
if name == "cancel_gesture":
|
||||
cancel_seen.set()
|
||||
return {"success": True, "execution": "observed", "code": "CANCEL_RECORDED"}
|
||||
started.set()
|
||||
try:
|
||||
if not release.wait(2):
|
||||
raise RuntimeError("test transport was not released")
|
||||
return {"success": True, "execution": "completed", "code": "OK"}
|
||||
finally:
|
||||
finished.set()
|
||||
|
||||
driver = object.__new__(AndroidAdbDriver)
|
||||
driver._ui_adb_client = SimpleNamespace(native_request=request)
|
||||
operation = asyncio.create_task(driver.native_request(command, {"operation_id": "test-cancel"}))
|
||||
try:
|
||||
assert await asyncio.to_thread(started.wait, 1)
|
||||
operation.cancel()
|
||||
done, _ = await asyncio.wait({operation}, timeout=0.05)
|
||||
if command == "inspect_control":
|
||||
assert operation in done and not finished.is_set()
|
||||
else:
|
||||
assert operation not in done and not finished.is_set()
|
||||
operation.cancel()
|
||||
done, _ = await asyncio.wait({operation}, timeout=0.02)
|
||||
assert not done
|
||||
assert cancel_seen.is_set() == (command == "touch_gesture")
|
||||
finally:
|
||||
release.set()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await operation
|
||||
assert await asyncio.to_thread(finished.wait, 1)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancelled_input_transport_failure_does_not_replace_cancellation():
|
||||
started, release = Event(), Event()
|
||||
|
||||
def request(*args, **kwargs):
|
||||
started.set()
|
||||
release.wait(2)
|
||||
raise OSError("response lost")
|
||||
|
||||
driver = object.__new__(AndroidAdbDriver)
|
||||
driver._ui_adb_client = SimpleNamespace(native_request=request)
|
||||
operation = asyncio.create_task(driver.native_request("control_action", {}))
|
||||
try:
|
||||
assert await asyncio.to_thread(started.wait, 1)
|
||||
operation.cancel()
|
||||
await asyncio.sleep(0)
|
||||
finally:
|
||||
release.set()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await operation
|
||||
@@ -85,6 +85,12 @@ def test_tool_declarations_match_fixture():
|
||||
"SWIPE_TOOL": "swipe",
|
||||
"DRAG_TOOL": "drag",
|
||||
"ACTION_SEQUENCE_TOOL": "action_sequence",
|
||||
"PINCH_TOOL": "pinch",
|
||||
"TOUCH_GESTURE_TOOL": "touch_gesture",
|
||||
"READ_CONTROL_TOOL": "read_control",
|
||||
"CONTROL_ACTION_TOOL": "control_action",
|
||||
"WAIT_UNTIL_TOOL": "wait_until",
|
||||
"SCROLL_TO_TOOL": "scroll_to",
|
||||
"PRESS_KEY_TOOL": "press_key",
|
||||
"MANAGE_APP_TOOL": "manage_app",
|
||||
"WAIT_FOR_DELAY_TOOL": "wait_for_delay",
|
||||
@@ -125,6 +131,12 @@ def test_validator_declaration_order_is_stable():
|
||||
"swipe",
|
||||
"drag",
|
||||
"action_sequence",
|
||||
"pinch",
|
||||
"touch_gesture",
|
||||
"read_control",
|
||||
"control_action",
|
||||
"wait_until",
|
||||
"scroll_to",
|
||||
"press_key",
|
||||
"read_note",
|
||||
"list_notes",
|
||||
|
||||
@@ -0,0 +1,404 @@
|
||||
"""Native action input contracts and geometry, without dispatching device input."""
|
||||
|
||||
from copy import deepcopy
|
||||
import math
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from artemis.mcp.native_actions import (
|
||||
NATIVE_REQUESTS,
|
||||
ControlActionRequest,
|
||||
ControlSelector,
|
||||
PinchRequest,
|
||||
TouchGestureRequest,
|
||||
WaitUntilRequest,
|
||||
plan_pinch,
|
||||
plan_touch,
|
||||
validate_native_action,
|
||||
)
|
||||
|
||||
|
||||
def touch(**overrides):
|
||||
return {
|
||||
"frames": [
|
||||
{"at_ms": 0, "points": [[100, 200]]},
|
||||
{"at_ms": 400, "points": [[300, 600]]},
|
||||
],
|
||||
"coordinate_space": "pixel",
|
||||
"target_description": "observed canvas",
|
||||
**overrides,
|
||||
}
|
||||
|
||||
|
||||
def pinch(**overrides):
|
||||
return {
|
||||
"center": [200, 400],
|
||||
"start_span": 100,
|
||||
"end_span": 200,
|
||||
"coordinate_space": "pixel",
|
||||
"target_description": "observed canvas",
|
||||
**overrides,
|
||||
}
|
||||
|
||||
|
||||
VALID_REQUESTS = {
|
||||
"pinch": pinch(),
|
||||
"touch_gesture": touch(),
|
||||
"read_control": {"selector": {"resource_id": "fixture:id/range"}},
|
||||
"control_action": {
|
||||
"selector": {"resource_id": "fixture:id/range"},
|
||||
"action": "set_progress",
|
||||
"value": 37.5,
|
||||
},
|
||||
"wait_until": {"selector": {"text": "Ready"}, "state": "visible"},
|
||||
"scroll_to": {
|
||||
"container": {"resource_id": "fixture:id/list"},
|
||||
"target": {"text": "Destination"},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", VALID_REQUESTS)
|
||||
def test_every_native_request_validates_before_dispatch_and_forbids_extra_fields(name):
|
||||
args = deepcopy(VALID_REQUESTS[name])
|
||||
assert validate_native_action(name, args, 400, 800).model_dump(exclude_none=True)
|
||||
with pytest.raises(ValidationError):
|
||||
validate_native_action(name, {**args, "coordinate_spce": "pixel"}, 400, 800)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"selector",
|
||||
[
|
||||
{},
|
||||
{"bounds": [0, 0, 10, 10]},
|
||||
{"text": ""},
|
||||
{"text": 123},
|
||||
{"resource_id": "item", "resourceId": "typo"},
|
||||
{"resource_id": "item", "bounds": [0, 0, 10]},
|
||||
{"resource_id": "item", "bounds": [0, 0, 10, 10, 20]},
|
||||
{"resource_id": "item", "bounds": [10, 0, 10, 20]},
|
||||
{"resource_id": "item", "bounds": [20, 0, 10, 20]},
|
||||
{"resource_id": "item", "bounds": [0, 0, 10, 0]},
|
||||
{"resource_id": "item", "bounds": [False, 0, 10, 10]},
|
||||
{"resource_id": "item", "bounds": [0, 0, 10.0, 10]},
|
||||
{"resource_id": "item", "bounds": [0, 0, "10", 10]},
|
||||
],
|
||||
)
|
||||
def test_selectors_require_observed_semantics_and_strict_nonempty_bounds(selector):
|
||||
with pytest.raises(ValidationError):
|
||||
ControlSelector.model_validate(selector)
|
||||
|
||||
|
||||
def test_exact_selector_strings_keep_whitespace_and_do_not_choose_one_attribute():
|
||||
observed = {
|
||||
"resource_id": "fixture:id/editor",
|
||||
"text": " A😀B \n",
|
||||
"content_desc": " description \t",
|
||||
"class_name": "android.widget.EditText",
|
||||
"package_name": "fixture",
|
||||
"bounds": [-5, 20, 100, 60],
|
||||
}
|
||||
assert ControlSelector.model_validate(observed).model_dump(exclude_none=True) == observed
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"frames",
|
||||
[
|
||||
[{"at_ms": 0, "points": [[1, 2]]}],
|
||||
[{"at_ms": i, "points": [[1, 2]]} for i in range(33)],
|
||||
[{"at_ms": 1, "points": [[1, 2]]}, {"at_ms": 2, "points": [[2, 3]]}],
|
||||
[{"at_ms": 0, "points": [[1, 2]]}, {"at_ms": 0, "points": [[2, 3]]}],
|
||||
[{"at_ms": 0, "points": [[1, 2]]}, {"at_ms": -1, "points": [[2, 3]]}],
|
||||
[{"at_ms": 0, "points": [[1, 2]]}, {"at_ms": 10001, "points": [[2, 3]]}],
|
||||
[{"at_ms": 0, "points": []}, {"at_ms": 10, "points": []}],
|
||||
[{"at_ms": 0, "points": [[1, 2]] * 6}, {"at_ms": 10, "points": [[2, 3]] * 6}],
|
||||
[{"at_ms": 0, "points": [[1, 2]]}, {"at_ms": 10, "points": [[2, 3], [4, 5]]}],
|
||||
[{"at_ms": 0, "points": [[1]]}, {"at_ms": 10, "points": [[2, 3]]}],
|
||||
[{"at_ms": 0, "points": [[1, 2, 3]]}, {"at_ms": 10, "points": [[2, 3]]}],
|
||||
[{"at_ms": 0, "points": [[True, 2]]}, {"at_ms": 10, "points": [[2, 3]]}],
|
||||
[{"at_ms": 0, "points": [[1.0, 2]]}, {"at_ms": 10, "points": [[2, 3]]}],
|
||||
[{"at_ms": 0, "points": [["1", 2]]}, {"at_ms": 10, "points": [[2, 3]]}],
|
||||
[{"at_ms": 0, "points": [[1, 2]]}, {"at_ms": "10", "points": [[2, 3]]}],
|
||||
[{"at_ms": 0, "points": [[1, 2]]}, {"at_ms": 10.0, "points": [[2, 3]]}],
|
||||
[{"at_ms": 0, "points": [[1, 2]]}, {"at_ms": True, "points": [[2, 3]]}],
|
||||
[
|
||||
{"at_ms": 0, "points": [[1, 2]], "pressure": 0.5},
|
||||
{"at_ms": 10, "points": [[2, 3]]},
|
||||
],
|
||||
],
|
||||
)
|
||||
def test_touch_timelines_and_pointer_shapes_are_strict(frames):
|
||||
with pytest.raises(ValidationError):
|
||||
TouchGestureRequest.model_validate(touch(frames=frames))
|
||||
|
||||
|
||||
def test_pixel_keyframes_preserve_holds_and_every_intermediate_segment():
|
||||
frames = [
|
||||
{"at_ms": 0, "points": [[191, 501], [211, 502]]},
|
||||
{"at_ms": 150, "points": [[191, 501], [211, 502]]},
|
||||
{"at_ms": 900, "points": [[192, 532], [212, 533]]},
|
||||
]
|
||||
planned = plan_touch(TouchGestureRequest.model_validate(touch(frames=frames)), 400, 800)
|
||||
assert planned["frames"] == frames
|
||||
assert planned["coordinate_space"] == "pixel"
|
||||
assert planned["screen_size"] == [400, 800]
|
||||
|
||||
|
||||
def test_full_supported_pointer_count_frame_count_and_duration_remain_valid():
|
||||
frames = [
|
||||
{
|
||||
"at_ms": round(frame * 10000 / 31),
|
||||
"points": [[50 + finger * 30, 100 + frame] for finger in range(5)],
|
||||
}
|
||||
for frame in range(32)
|
||||
]
|
||||
request = validate_native_action("touch_gesture", touch(frames=frames), 400, 800)
|
||||
assert plan_touch(request, 400, 800)["frames"] == frames
|
||||
|
||||
|
||||
def test_normalized_points_use_each_display_axis_and_accept_the_nominal_edges():
|
||||
request = TouchGestureRequest.model_validate(
|
||||
touch(
|
||||
coordinate_space="normalized",
|
||||
frames=[
|
||||
{"at_ms": 0, "points": [[0, 0]]},
|
||||
{"at_ms": 100, "points": [[500, 500]]},
|
||||
{"at_ms": 200, "points": [[1000, 1000]]},
|
||||
],
|
||||
)
|
||||
)
|
||||
assert [f["points"] for f in plan_touch(request, 720, 1600)["frames"]] == [
|
||||
[[0, 0]],
|
||||
[[360, 800]],
|
||||
[[719, 1599]],
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"space,point",
|
||||
[
|
||||
("pixel", [-1, 0]),
|
||||
("pixel", [400, 0]),
|
||||
("pixel", [0, 800]),
|
||||
("normalized", [-1, 500]),
|
||||
("normalized", [1001, 500]),
|
||||
("normalized", [500, 1001]),
|
||||
],
|
||||
)
|
||||
def test_offscreen_paths_fail_before_any_coordinate_can_be_clamped(space, point):
|
||||
args = touch(
|
||||
coordinate_space=space,
|
||||
frames=[{"at_ms": 0, "points": [[100, 100]]}, {"at_ms": 100, "points": [point]}],
|
||||
)
|
||||
with pytest.raises(ValueError, match="outside"):
|
||||
validate_native_action("touch_gesture", args, 400, 800)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dimensions", [(0, 800), (400, 0), (-1, 800)])
|
||||
def test_invalid_display_dimensions_cannot_produce_a_plan(dimensions):
|
||||
with pytest.raises(ValueError, match="positive"):
|
||||
validate_native_action("touch_gesture", touch(), *dimensions)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"before,after",
|
||||
[
|
||||
([[100, 100], [300, 100]], [[300, 100], [100, 100]]),
|
||||
([[100, 100], [300, 100]], [[200, 200], [200, 200]]),
|
||||
([[100, 100], [100, 100]], [[100, 200], [300, 200]]),
|
||||
],
|
||||
)
|
||||
def test_fingers_may_not_merge_at_frames_or_between_frames(before, after):
|
||||
with pytest.raises(ValueError, match="remain distinct"):
|
||||
validate_native_action(
|
||||
"touch_gesture",
|
||||
touch(frames=[{"at_ms": 0, "points": before}, {"at_ms": 100, "points": after}]),
|
||||
400,
|
||||
800,
|
||||
)
|
||||
|
||||
|
||||
def test_paths_can_cross_geometrically_when_fingers_reach_the_crossing_at_different_times():
|
||||
args = touch(
|
||||
frames=[
|
||||
{"at_ms": 0, "points": [[0, 100], [100, 0]]},
|
||||
{"at_ms": 400, "points": [[200, 100], [100, 400]]},
|
||||
]
|
||||
)
|
||||
assert (
|
||||
plan_touch(TouchGestureRequest.model_validate(args), 400, 800)["frames"] == args["frames"]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dimensions", [(720, 1600), (1600, 720)])
|
||||
def test_normalized_pinch_spans_use_short_edge_without_stretching_diagonal_angles(dimensions):
|
||||
width, height = dimensions
|
||||
request = PinchRequest.model_validate(
|
||||
pinch(
|
||||
center=[500, 500],
|
||||
start_span=200,
|
||||
end_span=400,
|
||||
angle_degrees=45,
|
||||
coordinate_space="normalized",
|
||||
)
|
||||
)
|
||||
planned = plan_pinch(request, width, height)
|
||||
for frame, expected_span in zip(planned["frames"], [144, 288], strict=True):
|
||||
first, second = frame["points"]
|
||||
assert second[0] - first[0] == second[1] - first[1]
|
||||
assert math.dist(first, second) == pytest.approx(expected_span, abs=1.5)
|
||||
assert [(first[i] + second[i]) / 2 for i in (0, 1)] == [width / 2, height / 2]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("rotation", [-360, -180, -90, 90, 180, 360])
|
||||
def test_rotation_uses_curved_synchronized_keyframes_and_keeps_both_fingers_down(rotation):
|
||||
planned = plan_pinch(
|
||||
PinchRequest.model_validate(
|
||||
pinch(center=[500, 500], start_span=200, end_span=200, rotation_degrees=rotation)
|
||||
),
|
||||
1000,
|
||||
1000,
|
||||
)
|
||||
frames = planned["frames"]
|
||||
assert len(frames) == abs(rotation) // 15 + 1
|
||||
assert frames[0]["points"] == [[400, 500], [600, 500]]
|
||||
assert frames[0]["at_ms"] == 0 and frames[-1]["at_ms"] == 800
|
||||
assert all(a["at_ms"] < b["at_ms"] for a, b in zip(frames, frames[1:]))
|
||||
assert all(len(frame["points"]) == 2 for frame in frames)
|
||||
quarter_turn = frames[6]["points"][1]
|
||||
assert quarter_turn == [500, 600 if rotation > 0 else 400]
|
||||
for frame in frames:
|
||||
assert math.dist(*frame["points"]) == pytest.approx(200, abs=1.5)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"overrides",
|
||||
[
|
||||
{"start_span": 0},
|
||||
{"end_span": -1},
|
||||
{"start_span": True},
|
||||
{"end_span": "200"},
|
||||
{"end_span": 100},
|
||||
{"rotation_degrees": 361},
|
||||
{"angle_degrees": float("nan")},
|
||||
{"angle_degrees": float("inf")},
|
||||
{"duration_ms": 0},
|
||||
{"duration_ms": 10001},
|
||||
{"duration_ms": 1.0},
|
||||
{"duration_ms": 20, "rotation_degrees": 360},
|
||||
{"center": [0, 0]},
|
||||
{"coordinate_space": "screen"},
|
||||
],
|
||||
)
|
||||
def test_invalid_pinch_geometry_and_units_fail_during_planning(overrides):
|
||||
with pytest.raises(ValueError):
|
||||
validate_native_action("pinch", pinch(**overrides), 400, 800)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("action", ["set_text", "replace_selection"])
|
||||
@pytest.mark.parametrize("text", ["", " A😀B \n\t"])
|
||||
def test_precise_text_edits_preserve_empty_input_unicode_and_whitespace(action, text):
|
||||
request = ControlActionRequest.model_validate(
|
||||
{"selector": {"resource_id": "editor"}, "action": action, "text": text}
|
||||
)
|
||||
assert request.model_dump(mode="json", exclude_none=True)["text"] == text
|
||||
|
||||
|
||||
def test_selection_offsets_are_preserved_as_utf16_units_without_python_character_conversion():
|
||||
text = "A😀B"
|
||||
end_of_emoji = len(text[:2].encode("utf-16-le")) // 2
|
||||
request = ControlActionRequest.model_validate(
|
||||
{
|
||||
"selector": {"text": text},
|
||||
"action": "set_selection",
|
||||
"selection_start": 1,
|
||||
"selection_end": end_of_emoji,
|
||||
}
|
||||
)
|
||||
wire = request.model_dump(mode="json", exclude_none=True)
|
||||
assert (wire["selection_start"], wire["selection_end"]) == (1, 3)
|
||||
# Bounds and surrogate-pair boundaries require the fresh native text, checked by Java.
|
||||
assert end_of_emoji != len(text[:2])
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"args",
|
||||
[
|
||||
{"action": "click", "value": 1},
|
||||
{"action": "click", "text": "extra"},
|
||||
{"action": "click", "selection_start": 0, "selection_end": 1},
|
||||
{"action": "set_text"},
|
||||
{"action": "set_text", "text": "x" * 65537},
|
||||
{"action": "replace_selection"},
|
||||
{"action": "set_progress"},
|
||||
{"action": "set_progress", "value": True},
|
||||
{"action": "set_progress", "value": "3"},
|
||||
{"action": "set_progress", "value": float("nan")},
|
||||
{"action": "set_progress", "value": float("inf")},
|
||||
{"action": "set_selection", "selection_start": 0},
|
||||
{"action": "set_selection", "selection_start": -1, "selection_end": 1},
|
||||
{"action": "set_selection", "selection_start": 2, "selection_end": 1},
|
||||
{"action": "set_selection", "selection_start": False, "selection_end": 1},
|
||||
{"action": "set_selection", "selection_start": 0, "selection_end": 1.0},
|
||||
{"action": "set_selection", "selection_start": 0, "selection_end": "1"},
|
||||
{"action": "click", "expected_token": "a" * 63},
|
||||
{"action": "click", "expected_token": "G" * 64},
|
||||
{"action": "toggle"},
|
||||
],
|
||||
)
|
||||
def test_control_actions_reject_coercion_wrong_arguments_and_invalid_tokens(args):
|
||||
with pytest.raises(ValidationError):
|
||||
ControlActionRequest.model_validate({"selector": {"resource_id": "control"}, **args})
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"state,extra",
|
||||
[
|
||||
("visible", {}),
|
||||
("absent", {}),
|
||||
("enabled", {}),
|
||||
("disabled", {}),
|
||||
("checked", {}),
|
||||
("unchecked", {}),
|
||||
("selected", {}),
|
||||
("unselected", {}),
|
||||
("text", {"expected_text": " exact 😀 \n"}),
|
||||
("value", {"expected_value": 0}),
|
||||
],
|
||||
)
|
||||
def test_nonstable_waits_allow_short_deadlines_without_overriding_stability_default(state, extra):
|
||||
request = WaitUntilRequest.model_validate(
|
||||
{"selector": {"resource_id": "control"}, "state": state, "timeout_ms": 100, **extra}
|
||||
)
|
||||
assert request.timeout_ms == 100 and request.stable_ms == 300
|
||||
if state == "text":
|
||||
assert request.expected_text == extra["expected_text"]
|
||||
|
||||
|
||||
def test_stability_window_must_fit_the_deadline_only_for_stable_waits():
|
||||
args = {"selector": {"resource_id": "control"}, "state": "stable", "timeout_ms": 100}
|
||||
with pytest.raises(ValidationError, match="stable_ms must fit"):
|
||||
WaitUntilRequest.model_validate(args)
|
||||
assert WaitUntilRequest.model_validate({**args, "stable_ms": 100}).stable_ms == 100
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"name,args",
|
||||
[
|
||||
("wait_until", {"state": "text"}),
|
||||
("wait_until", {"state": "value"}),
|
||||
("wait_until", {"state": "visible", "expected_text": "wrong predicate"}),
|
||||
("wait_until", {"state": "visible", "tolerance": 1}),
|
||||
("wait_until", {"state": "value", "expected_value": float("inf")}),
|
||||
("wait_until", {"state": "visible", "timeout_ms": 0}),
|
||||
("wait_until", {"state": "visible", "poll_interval_ms": 49}),
|
||||
("scroll_to", {"max_scrolls": 0}),
|
||||
("scroll_to", {"max_scrolls": 21}),
|
||||
("scroll_to", {"timeout_ms": 100, "settle_ms": 200}),
|
||||
("scroll_to", {"direction": "diagonal"}),
|
||||
],
|
||||
)
|
||||
def test_condition_and_scroll_requests_reject_incompatible_or_unbounded_arguments(name, args):
|
||||
with pytest.raises(ValidationError):
|
||||
NATIVE_REQUESTS[name].model_validate({**VALID_REQUESTS[name], **args})
|
||||
@@ -0,0 +1,775 @@
|
||||
"""Native semantics with deterministic clocks and no devices or paid models."""
|
||||
|
||||
import asyncio
|
||||
from copy import deepcopy
|
||||
from unittest.mock import AsyncMock
|
||||
from uuid import UUID
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.mcp import native_executor
|
||||
from artemis.mcp.native_executor import execute_native
|
||||
|
||||
SELECTOR = {"resource_id": "control"}
|
||||
CONTAINER = {"resource_id": "list"}
|
||||
TARGET = {"text": "destination"}
|
||||
TOKEN = "a" * 64
|
||||
|
||||
|
||||
def node(**overrides):
|
||||
return {
|
||||
"token": TOKEN,
|
||||
"content_signature": "contents-a",
|
||||
"visible": True,
|
||||
"enabled": True,
|
||||
"checkable": True,
|
||||
"checked": False,
|
||||
"selected": False,
|
||||
"password": False,
|
||||
"text": "exact text",
|
||||
"actions": ["click", "set_progress", "set_text", "scroll_forward"],
|
||||
"range": {"min": 0, "max": 10, "current": 3, "type": 1},
|
||||
**overrides,
|
||||
}
|
||||
|
||||
|
||||
def observed(**overrides):
|
||||
return {"success": True, "execution": "observed", "code": "OK", "node": node(**overrides)}
|
||||
|
||||
|
||||
def missing(code="NOT_FOUND"):
|
||||
return {"success": False, "execution": "not_started", "code": code, "error": code}
|
||||
|
||||
|
||||
def completed(**extra):
|
||||
return {"success": True, "execution": "completed", "code": "OK", **extra}
|
||||
|
||||
|
||||
class Clock:
|
||||
def __init__(self):
|
||||
self.now = 0.0
|
||||
self.sleeps = []
|
||||
|
||||
def monotonic(self):
|
||||
return self.now
|
||||
|
||||
async def sleep(self, seconds):
|
||||
assert seconds > 0
|
||||
self.sleeps.append(seconds)
|
||||
self.now += seconds
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def clock(monkeypatch):
|
||||
fake = Clock()
|
||||
monkeypatch.setattr(native_executor, "monotonic", fake.monotonic)
|
||||
monkeypatch.setattr(native_executor, "sleep", fake.sleep)
|
||||
return fake
|
||||
|
||||
|
||||
async def run(name, args, native):
|
||||
return await execute_native(name, args, 400, 800, native)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("name", ["pinch", "touch_gesture"])
|
||||
async def test_all_gesture_geometry_is_validated_before_a_single_native_dispatch(name):
|
||||
args = (
|
||||
{
|
||||
"center": [200, 400],
|
||||
"start_span": 40,
|
||||
"end_span": 80,
|
||||
"rotation_degrees": 45,
|
||||
"duration_ms": 900,
|
||||
"coordinate_space": "pixel",
|
||||
"target_description": "observed control",
|
||||
}
|
||||
if name == "pinch"
|
||||
else {
|
||||
"frames": [
|
||||
{"at_ms": 0, "points": [[100, 100]]},
|
||||
{"at_ms": 300, "points": [[120, 160]]},
|
||||
{"at_ms": 900, "points": [[140, 220]]},
|
||||
],
|
||||
"coordinate_space": "pixel",
|
||||
"target_description": "observed control",
|
||||
}
|
||||
)
|
||||
native = AsyncMock(return_value=completed(native_marker="original receipt"))
|
||||
result = await run(name, args, native)
|
||||
|
||||
assert result["success"] and result["execution"] == "completed"
|
||||
assert result["verification"] == "pending_observation"
|
||||
assert result["native_receipt"]["native_marker"] == "original receipt"
|
||||
native.assert_awaited_once()
|
||||
command, sent = native.await_args.args
|
||||
assert command == "touch_gesture"
|
||||
assert UUID(sent["operation_id"])
|
||||
assert sent["frames"] == result["plan"]["frames"]
|
||||
assert sent["screen_size"] == [400, 800]
|
||||
assert native.await_args.kwargs == {"timeout": 5.9}
|
||||
|
||||
invalid = deepcopy(args)
|
||||
if name == "pinch":
|
||||
invalid["center"] = [0, 0]
|
||||
else:
|
||||
invalid["frames"][-1]["points"] = [[401, 220]]
|
||||
never = AsyncMock()
|
||||
refused = await run(name, invalid, never)
|
||||
assert not refused["success"] and refused["execution"] == "not_started"
|
||||
never.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_touch_timeout_never_replays_the_input():
|
||||
native = AsyncMock(side_effect=TimeoutError("reply lost"))
|
||||
result = await run(
|
||||
"touch_gesture",
|
||||
{
|
||||
"frames": [{"at_ms": 0, "points": [[10, 10]]}, {"at_ms": 20, "points": [[20, 30]]}],
|
||||
"target_description": "observed control",
|
||||
"require_visible": "required control",
|
||||
},
|
||||
native,
|
||||
)
|
||||
assert not result["success"] and result["execution"] == "unknown"
|
||||
assert result["code"] == "TIMEOUT"
|
||||
native.assert_awaited_once()
|
||||
assert native.await_args.args[1]["require_visible"] == "required control"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_read_control_keeps_native_evidence_and_omits_null_selector_fields():
|
||||
receipt = observed()
|
||||
native = AsyncMock(return_value=receipt)
|
||||
result = await run("read_control", {"selector": {**SELECTOR, "text": None}}, native)
|
||||
|
||||
native.assert_awaited_once_with("inspect_control", {"selector": SELECTOR}, timeout=None)
|
||||
assert result["node"] == receipt["node"]
|
||||
assert result["native_receipt"] == receipt
|
||||
assert result["execution"] == "observed" and result["action"] == "read_control"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_control_action_uses_fresh_token_preserves_whitespace_and_does_not_verify_business_value():
|
||||
before = observed()
|
||||
after = node(text="different accepted text")
|
||||
receipt = completed(node_before=before["node"], node_after=after)
|
||||
native = AsyncMock(side_effect=[before, receipt])
|
||||
result = await run(
|
||||
"control_action", {"selector": SELECTOR, "action": "set_text", "text": " exact\n"}, native
|
||||
)
|
||||
|
||||
assert native.await_args_list[1].args == (
|
||||
"control_action",
|
||||
{"selector": SELECTOR, "action": "set_text", "text": " exact\n", "expected_token": TOKEN},
|
||||
)
|
||||
assert native.await_count == 2
|
||||
assert result["success"] and result["verification"] == "pending_observation"
|
||||
assert result["node_before"] == before["node"]
|
||||
assert result["node_after"] == after
|
||||
assert result["native_receipt"] == receipt
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"snapshot,request_extra,code",
|
||||
[
|
||||
(node(), {"expected_token": "b" * 64}, "STALE"),
|
||||
(node(actions=["long_click"]), {}, "UNSUPPORTED"),
|
||||
(node(enabled=False), {}, "DISABLED"),
|
||||
(node(token=None), {}, "INVALID_RESPONSE"),
|
||||
],
|
||||
)
|
||||
async def test_control_action_refuses_stale_unsupported_or_disabled_without_input(
|
||||
snapshot, request_extra, code
|
||||
):
|
||||
native = AsyncMock(return_value={**observed(), "node": snapshot})
|
||||
result = await run(
|
||||
"control_action", {"selector": SELECTOR, "action": "click", **request_extra}, native
|
||||
)
|
||||
assert not result["success"] and result["code"] == code
|
||||
assert result["execution"] == "not_started"
|
||||
assert result["node_before"] == snapshot
|
||||
native.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"native_range,value",
|
||||
[
|
||||
({"min": 0, "max": 10, "current": 3}, 11),
|
||||
({"min": 0, "max": 10, "current": float("nan")}, 3),
|
||||
({"min": 0, "max": 10, "current": 3, "type": 0}, 3.5),
|
||||
],
|
||||
)
|
||||
async def test_set_progress_refuses_invalid_live_range_before_input(native_range, value):
|
||||
native = AsyncMock(return_value=observed(range=native_range))
|
||||
result = await run(
|
||||
"control_action", {"selector": SELECTOR, "action": "set_progress", "value": value}, native
|
||||
)
|
||||
assert not result["success"] and result["execution"] == "not_started"
|
||||
native.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_control_transport_failure_is_unknown_and_never_replayed():
|
||||
native = AsyncMock(side_effect=[observed(), OSError("lost reply")])
|
||||
result = await run("control_action", {"selector": SELECTOR, "action": "click"}, native)
|
||||
assert result["code"] == "NATIVE_ERROR" and result["execution"] == "unknown"
|
||||
assert not result["success"] and native.await_count == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"receipt",
|
||||
[
|
||||
"legacy unstructured success",
|
||||
None,
|
||||
{"success": True, "execution": [], "code": "OK"},
|
||||
{"success": True, "execution": "unknown", "code": "OK"},
|
||||
{"success": True, "execution": "completed"},
|
||||
],
|
||||
)
|
||||
async def test_malformed_write_receipts_are_unknown_and_never_replayed(receipt):
|
||||
native = AsyncMock(side_effect=[observed(), receipt])
|
||||
result = await run("control_action", {"selector": SELECTOR, "action": "click"}, native)
|
||||
assert not result["success"] and result["execution"] == "unknown"
|
||||
assert result["code"] == "INVALID_RESPONSE" and native.await_count == 2
|
||||
assert result["native_receipt"] == receipt
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wait_observes_delayed_condition_without_operating_or_sleeping_after_success(clock):
|
||||
native = AsyncMock(side_effect=[missing(), observed(visible=False), observed()])
|
||||
result = await run(
|
||||
"wait_until",
|
||||
{"selector": SELECTOR, "state": "visible", "timeout_ms": 1000, "poll_interval_ms": 100},
|
||||
native,
|
||||
)
|
||||
assert result["success"] and result["execution"] == "observed"
|
||||
assert result["attempts"] == 3 and clock.now == pytest.approx(0.2)
|
||||
assert [call.args[0] for call in native.await_args_list] == ["inspect_control"] * 3
|
||||
assert [call.kwargs["timeout"] for call in native.await_args_list] == pytest.approx(
|
||||
[1, 0.9, 0.8]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("changed_field", ["token", "content_signature"])
|
||||
async def test_stability_restarts_on_identity_or_descendant_redraw(clock, changed_field):
|
||||
change = {changed_field: "changed"}
|
||||
native = AsyncMock(
|
||||
side_effect=[observed(), observed(**change), observed(**change), observed(**change)]
|
||||
)
|
||||
result = await run(
|
||||
"wait_until",
|
||||
{
|
||||
"selector": SELECTOR,
|
||||
"state": "stable",
|
||||
"stable_ms": 190,
|
||||
"timeout_ms": 1000,
|
||||
"poll_interval_ms": 100,
|
||||
},
|
||||
native,
|
||||
)
|
||||
assert result["success"] and result["attempts"] == 4
|
||||
assert clock.now == pytest.approx(0.3)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_zero_stability_window_still_requires_two_actual_reads(clock):
|
||||
native = AsyncMock(return_value=observed())
|
||||
result = await run(
|
||||
"wait_until", {"selector": SELECTOR, "state": "stable", "stable_ms": 0}, native
|
||||
)
|
||||
assert result["success"] and result["attempts"] == 2
|
||||
assert len(clock.sleeps) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("receipt", [missing(), observed(visible=False)])
|
||||
async def test_absent_accepts_only_not_found_or_unique_explicitly_invisible(clock, receipt):
|
||||
native = AsyncMock(return_value=receipt)
|
||||
result = await run("wait_until", {"selector": SELECTOR, "state": "absent"}, native)
|
||||
assert result["success"] and result["execution"] == "observed"
|
||||
native.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"error", ["AMBIGUOUS", "UNSUPPORTED", "LIMIT_EXCEEDED", "INVALID_RESPONSE"]
|
||||
)
|
||||
async def test_absent_rejects_nontransient_inspection_errors_immediately(clock, error):
|
||||
native = AsyncMock(return_value=missing(error))
|
||||
result = await run("wait_until", {"selector": SELECTOR, "state": "absent"}, native)
|
||||
assert not result["success"] and result["code"] == error
|
||||
native.assert_awaited_once()
|
||||
assert not clock.sleeps
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("error", ["UNAVAILABLE", "READ_FAILED"])
|
||||
async def test_unreadable_windows_time_out_without_proving_absence(clock, error):
|
||||
native = AsyncMock(return_value=missing(error))
|
||||
result = await run(
|
||||
"wait_until",
|
||||
{"selector": SELECTOR, "state": "absent", "timeout_ms": 300, "poll_interval_ms": 100},
|
||||
native,
|
||||
)
|
||||
assert not result["success"] and result["code"] == "TIMEOUT"
|
||||
assert result["native_receipt"]["code"] == error
|
||||
assert result["attempts"] == 3 and clock.now == pytest.approx(0.3)
|
||||
assert [sample["code"] for sample in result["observations"]] == [error] * 3
|
||||
assert [call.args[0] for call in native.await_args_list] == ["inspect_control"] * 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("error", ["UNAVAILABLE", "READ_FAILED"])
|
||||
@pytest.mark.parametrize("state,receipt", [("visible", observed()), ("absent", missing())])
|
||||
async def test_wait_recovers_only_after_a_reliable_observation(clock, error, state, receipt):
|
||||
native = AsyncMock(side_effect=[missing(error), receipt])
|
||||
result = await run(
|
||||
"wait_until",
|
||||
{"selector": SELECTOR, "state": state, "timeout_ms": 500, "poll_interval_ms": 100},
|
||||
native,
|
||||
)
|
||||
assert result["success"] and result["execution"] == "observed"
|
||||
assert result["attempts"] == 2 and clock.now == pytest.approx(0.1)
|
||||
assert result["observations"][0]["code"] == error
|
||||
assert result["native_receipt"] == receipt
|
||||
assert [call.args[0] for call in native.await_args_list] == ["inspect_control"] * 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("error", ["UNAVAILABLE", "READ_FAILED"])
|
||||
async def test_transient_read_failure_restarts_the_full_stability_window(clock, error):
|
||||
native = AsyncMock(side_effect=[observed(), missing(error), observed(), observed(), observed()])
|
||||
result = await run(
|
||||
"wait_until",
|
||||
{
|
||||
"selector": SELECTOR,
|
||||
"state": "stable",
|
||||
"stable_ms": 190,
|
||||
"timeout_ms": 1000,
|
||||
"poll_interval_ms": 100,
|
||||
},
|
||||
native,
|
||||
)
|
||||
assert result["success"] and result["attempts"] == 5
|
||||
assert clock.now == pytest.approx(0.4)
|
||||
assert result["observations"][1]["code"] == error
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_absent_transport_failure_is_not_absence(clock):
|
||||
native = AsyncMock(side_effect=OSError("unavailable"))
|
||||
result = await run("wait_until", {"selector": SELECTOR, "state": "absent"}, native)
|
||||
assert not result["success"] and result["code"] == "NATIVE_ERROR"
|
||||
native.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("receipt", [{**observed(), "candidate_count": 2}, observed(visible=None)])
|
||||
async def test_absent_rejects_nonunique_or_unknown_visibility_observation(clock, receipt):
|
||||
native = AsyncMock(return_value=receipt)
|
||||
result = await run("wait_until", {"selector": SELECTOR, "state": "absent"}, native)
|
||||
assert not result["success"]
|
||||
assert result["code"] in {"AMBIGUOUS", "INVALID_STATE"}
|
||||
native.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"state,args,snapshot,success",
|
||||
[
|
||||
("text", {"expected_text": " exact text "}, node(), False),
|
||||
("text", {"expected_text": "exact text"}, node(), True),
|
||||
("value", {"expected_value": 3.1, "tolerance": 0.2}, node(), True),
|
||||
(
|
||||
"value",
|
||||
{"expected_value": 3},
|
||||
node(range={"min": 0, "max": float("inf"), "current": 3}),
|
||||
False,
|
||||
),
|
||||
("unchecked", {}, node(checkable=False, checked=False), False),
|
||||
("checked", {}, node(checked=True), True),
|
||||
("disabled", {}, node(enabled=False), True),
|
||||
("unselected", {}, node(selected=False), True),
|
||||
],
|
||||
)
|
||||
async def test_wait_predicates_use_meaningful_exact_native_values(
|
||||
clock, state, args, snapshot, success
|
||||
):
|
||||
native = AsyncMock(return_value={**observed(), "node": snapshot})
|
||||
result = await run(
|
||||
"wait_until",
|
||||
{"selector": SELECTOR, "state": state, "timeout_ms": 400, "stable_ms": 0, **args},
|
||||
native,
|
||||
)
|
||||
assert result["success"] is success
|
||||
assert all(call.args[0] == "inspect_control" for call in native.await_args_list)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wait_deadline_is_bounded_and_preserves_observations(clock):
|
||||
native = AsyncMock(return_value=observed(visible=False))
|
||||
result = await run(
|
||||
"wait_until",
|
||||
{"selector": SELECTOR, "state": "visible", "timeout_ms": 350, "poll_interval_ms": 100},
|
||||
native,
|
||||
)
|
||||
assert not result["success"] and result["code"] == "TIMEOUT"
|
||||
assert clock.now == pytest.approx(0.35) and native.await_count == 4
|
||||
assert len(result["observations"]) == 4
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wait_rejects_condition_returned_after_monotonic_deadline(clock):
|
||||
async def native(command, args, *, timeout=None):
|
||||
clock.now += 0.6
|
||||
return observed()
|
||||
|
||||
result = await run(
|
||||
"wait_until", {"selector": SELECTOR, "state": "visible", "timeout_ms": 500}, native
|
||||
)
|
||||
assert not result["success"] and result["code"] == "TIMEOUT"
|
||||
assert result["attempts"] == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wait_stops_a_nonreturning_native_read_at_its_deadline(clock):
|
||||
calls = []
|
||||
|
||||
async def native(command, args, *, timeout=None):
|
||||
calls.append(command)
|
||||
await asyncio.Event().wait()
|
||||
|
||||
result = await run(
|
||||
"wait_until",
|
||||
{"selector": SELECTOR, "state": "visible", "timeout_ms": 20, "stable_ms": 0},
|
||||
native,
|
||||
)
|
||||
assert not result["success"] and result["code"] == "TIMEOUT"
|
||||
assert calls == ["inspect_control"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_long_wait_bounds_evidence_memory_while_retaining_first_and_last_reads(clock):
|
||||
native = AsyncMock(return_value=observed(visible=False))
|
||||
result = await run(
|
||||
"wait_until",
|
||||
{"selector": SELECTOR, "state": "visible", "timeout_ms": 4000, "poll_interval_ms": 50},
|
||||
native,
|
||||
)
|
||||
assert result["attempts"] > 64 and len(result["observations"]) == 64
|
||||
assert result["observations"][0]["at_ms"] == 0
|
||||
assert result["observations"][-1]["at_ms"] > 3000
|
||||
|
||||
|
||||
class ScrollNative:
|
||||
def __init__(self, clock, *, delayed_target=None, progress=True, receipt=None):
|
||||
self.clock = clock
|
||||
self.delayed_target = delayed_target
|
||||
self.progress = progress
|
||||
self.receipt = receipt or completed()
|
||||
self.calls = []
|
||||
self.writes = 0
|
||||
|
||||
async def __call__(self, command, args, *, timeout=None):
|
||||
self.calls.append((command, deepcopy(args), timeout, self.clock.now))
|
||||
if command == "control_action":
|
||||
self.writes += 1
|
||||
return self.receipt
|
||||
if args["selector"] == TARGET:
|
||||
# An unscoped query would spuriously find a same-text external node.
|
||||
if args.get("scope") != CONTAINER:
|
||||
return observed(text="external false positive")
|
||||
if (
|
||||
self.writes
|
||||
and self.delayed_target is not None
|
||||
and self.clock.now >= self.delayed_target
|
||||
):
|
||||
return observed(text="destination")
|
||||
return missing()
|
||||
assert args == {"selector": CONTAINER}
|
||||
changed = self.writes and self.progress and self.clock.now >= 0.1
|
||||
return observed(
|
||||
token=("b" if self.writes else "a") * 64,
|
||||
content_signature=f"page-{self.writes}" if changed else "contents-a",
|
||||
)
|
||||
|
||||
|
||||
def scroll_request(**extra):
|
||||
return {"container": CONTAINER, "target": TARGET, "settle_ms": 300, "timeout_ms": 2000, **extra}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scroll_waits_for_delayed_scoped_target_after_content_changes(clock):
|
||||
native = ScrollNative(clock, delayed_target=0.2)
|
||||
result = await run("scroll_to", scroll_request(), native)
|
||||
assert result["success"] and result["execution"] == "completed"
|
||||
assert native.writes == 1 and clock.now >= 0.4 - 1e-9
|
||||
assert result["node"]["text"] == "destination"
|
||||
assert result["scroll_results"][0]["native_receipt"]["success"]
|
||||
queries = [
|
||||
args
|
||||
for command, args, _, _ in native.calls
|
||||
if command == "inspect_control" and args["selector"] == TARGET
|
||||
]
|
||||
assert queries and all(query["scope"] == CONTAINER for query in queries)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scroll_token_change_without_content_change_is_no_progress(clock):
|
||||
native = ScrollNative(clock, progress=False)
|
||||
result = await run("scroll_to", scroll_request(), native)
|
||||
assert not result["success"] and result["code"] == "NO_PROGRESS"
|
||||
assert result["execution"] == "completed" and native.writes == 1
|
||||
assert clock.now == pytest.approx(0.5)
|
||||
assert result["scroll_results"][0]["native_receipt"]["success"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scroll_preserves_each_receipt_and_refreshes_token_with_hard_input_limit(clock):
|
||||
native = ScrollNative(clock)
|
||||
result = await run("scroll_to", scroll_request(max_scrolls=2), native)
|
||||
assert result["code"] == "MAX_SCROLLS" and not result["success"]
|
||||
assert native.writes == 2 and len(result["scroll_results"]) == 2
|
||||
writes = [(args, at) for command, args, _, at in native.calls if command == "control_action"]
|
||||
assert writes[0][0]["expected_token"] == TOKEN
|
||||
assert writes[1][0]["expected_token"] == "b" * 64
|
||||
assert writes[1][1] >= 0.3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("execution", ["unknown", "cancelled"])
|
||||
async def test_scroll_unknown_or_cancelled_input_is_never_repeated(clock, execution):
|
||||
receipt = {"success": False, "execution": execution, "code": "TIMEOUT", "native_marker": "kept"}
|
||||
native = ScrollNative(clock, receipt=receipt)
|
||||
result = await run("scroll_to", scroll_request(), native)
|
||||
assert not result["success"] and result["execution"] == execution
|
||||
assert result["scroll_results"][0]["native_receipt"] == receipt
|
||||
assert native.writes == 1 and not clock.sleeps
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scroll_boundary_rejection_is_no_progress_without_retry(clock):
|
||||
native = ScrollNative(clock, receipt=missing("ACTION_REJECTED"))
|
||||
result = await run("scroll_to", scroll_request(), native)
|
||||
assert result["code"] == "NO_PROGRESS" and native.writes == 1
|
||||
assert result["scroll_results"][0]["native_receipt"]["code"] == "ACTION_REJECTED"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("which", ["container", "target"])
|
||||
async def test_scroll_refuses_ambiguous_container_or_scoped_target_before_input(clock, which):
|
||||
calls = []
|
||||
|
||||
async def native(command, args, *, timeout=None):
|
||||
calls.append(command)
|
||||
assert command == "inspect_control"
|
||||
if args["selector"] == (CONTAINER if which == "container" else TARGET):
|
||||
return missing("AMBIGUOUS")
|
||||
return observed()
|
||||
|
||||
result = await run("scroll_to", scroll_request(), native)
|
||||
assert not result["success"] and result["code"] == "AMBIGUOUS"
|
||||
assert "control_action" not in calls
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"write_receipt,execution",
|
||||
[
|
||||
(completed(native_marker="late accepted input"), "completed"),
|
||||
(
|
||||
{
|
||||
"success": False,
|
||||
"execution": "unknown",
|
||||
"code": "TIMEOUT",
|
||||
"native_marker": "late uncertain receipt",
|
||||
},
|
||||
"unknown",
|
||||
),
|
||||
],
|
||||
)
|
||||
async def test_scroll_drains_dispatched_input_past_search_deadline_without_more_work(
|
||||
clock, write_receipt, execution
|
||||
):
|
||||
calls = []
|
||||
events = []
|
||||
pending_input = None
|
||||
|
||||
async def deliver_input():
|
||||
# Device input survives a lost HTTP waiter; cancelling its await cannot undo it.
|
||||
await asyncio.sleep(0.12)
|
||||
clock.now = 0.12
|
||||
events.append("input_delivered")
|
||||
return write_receipt
|
||||
|
||||
async def native(command, args, *, timeout=None):
|
||||
nonlocal pending_input
|
||||
calls.append((command, deepcopy(args), timeout))
|
||||
if command == "inspect_control":
|
||||
return missing() if args["selector"] == TARGET else observed()
|
||||
pending_input = asyncio.create_task(deliver_input())
|
||||
return await asyncio.wait_for(asyncio.shield(pending_input), timeout=timeout)
|
||||
|
||||
try:
|
||||
result = await run(
|
||||
"scroll_to", scroll_request(timeout_ms=50, settle_ms=50, max_scrolls=3), native
|
||||
)
|
||||
events.append("tool_returned")
|
||||
finally:
|
||||
if pending_input is not None:
|
||||
await pending_input
|
||||
|
||||
assert events == ["input_delivered", "tool_returned"]
|
||||
assert not result["success"] and result["code"] == "TIMEOUT"
|
||||
assert result["execution"] == execution
|
||||
assert result["scroll_results"][0]["native_receipt"] == write_receipt
|
||||
assert [command for command, _, _ in calls] == [
|
||||
"inspect_control",
|
||||
"inspect_control",
|
||||
"control_action",
|
||||
]
|
||||
assert calls[-1][2] == 5.0
|
||||
assert not clock.sleeps
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scroll_timeout_caps_total_search_even_when_content_changes(clock):
|
||||
native = ScrollNative(clock)
|
||||
result = await run("scroll_to", scroll_request(timeout_ms=350, settle_ms=300), native)
|
||||
assert result["code"] == "TIMEOUT" and not result["success"]
|
||||
assert clock.now == pytest.approx(0.35) and native.writes <= 2
|
||||
assert len(result["scroll_results"]) == native.writes
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scroll_requires_visible_target_not_merely_an_existing_scoped_node(clock):
|
||||
backend = ScrollNative(clock, delayed_target=0.2)
|
||||
|
||||
async def native(command, args, *, timeout=None):
|
||||
result = await backend(command, args, timeout=timeout)
|
||||
return (
|
||||
observed(text="destination", visible=False)
|
||||
if result.get("code") == "NOT_FOUND"
|
||||
else result
|
||||
)
|
||||
|
||||
result = await run("scroll_to", scroll_request(), native)
|
||||
assert result["success"] and backend.writes == 1
|
||||
assert clock.now >= 0.4 - 1e-9
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scroll_does_not_send_next_input_while_content_is_animating(clock):
|
||||
writes = []
|
||||
|
||||
async def native(command, args, *, timeout=None):
|
||||
if command == "control_action":
|
||||
writes.append(clock.now)
|
||||
return completed()
|
||||
if args["selector"] == TARGET:
|
||||
return missing()
|
||||
page = min(int(clock.now * 10 + 1e-6), 6) if writes else 0
|
||||
return observed(content_signature=f"animation-{page}")
|
||||
|
||||
result = await run("scroll_to", scroll_request(max_scrolls=2), native)
|
||||
assert result["code"] == "NO_PROGRESS" and len(writes) == 2
|
||||
assert writes[1] >= 0.9 - 1e-9
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scroll_transient_visible_target_during_animation_is_not_success(clock):
|
||||
writes = []
|
||||
|
||||
async def native(command, args, *, timeout=None):
|
||||
if command == "control_action":
|
||||
writes.append(clock.now)
|
||||
return completed()
|
||||
if args["selector"] == TARGET:
|
||||
return observed() if writes and 0.1 <= clock.now < 0.2 else missing()
|
||||
return observed(content_signature="after" if writes and clock.now >= 0.25 else "before")
|
||||
|
||||
result = await run("scroll_to", scroll_request(max_scrolls=1), native)
|
||||
assert result["code"] == "MAX_SCROLLS" and not result["success"]
|
||||
assert len(writes) == 1 and clock.now >= 0.55 - 1e-9
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scroll_tracks_target_outside_bounded_container_signature_sample(clock):
|
||||
writes = []
|
||||
|
||||
async def native(command, args, *, timeout=None):
|
||||
if command == "control_action":
|
||||
writes.append(clock.now)
|
||||
return completed()
|
||||
if args["selector"] == TARGET:
|
||||
return observed() if writes and 0.4 - 1e-9 <= clock.now < 0.45 - 1e-9 else missing()
|
||||
return observed(content_signature="after" if writes and clock.now >= 0.1 else "before")
|
||||
|
||||
result = await run("scroll_to", scroll_request(max_scrolls=1), native)
|
||||
assert result["code"] == "MAX_SCROLLS" and not result["success"]
|
||||
assert len(writes) == 1 and clock.now >= 0.75 - 1e-9
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_scroll_read_failure_after_accepted_input_preserves_receipt_without_more_input(clock):
|
||||
backend = ScrollNative(clock)
|
||||
|
||||
async def native(command, args, *, timeout=None):
|
||||
if command == "inspect_control" and backend.writes:
|
||||
raise OSError("read unavailable after accepted input")
|
||||
return await backend(command, args, timeout=timeout)
|
||||
|
||||
result = await run("scroll_to", scroll_request(), native)
|
||||
assert not result["success"] and result["execution"] == "completed"
|
||||
assert result["code"] == "NATIVE_ERROR" and backend.writes == 1
|
||||
assert result["scroll_results"][0]["native_receipt"]["success"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancelled_scroll_settling_never_sends_another_input(clock, monkeypatch):
|
||||
backend = ScrollNative(clock)
|
||||
|
||||
async def cancelled_sleep(seconds):
|
||||
raise asyncio.CancelledError
|
||||
|
||||
monkeypatch.setattr(native_executor, "sleep", cancelled_sleep)
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await run("scroll_to", scroll_request(), backend)
|
||||
assert backend.writes == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("name", ["control_action", "wait_until", "scroll_to", "touch_gesture"])
|
||||
async def test_cancellation_propagates_without_replaying_native_work(clock, name):
|
||||
requests = {
|
||||
"control_action": {"selector": SELECTOR, "action": "click"},
|
||||
"wait_until": {"selector": SELECTOR, "state": "visible"},
|
||||
"scroll_to": scroll_request(),
|
||||
"touch_gesture": {
|
||||
"frames": [{"at_ms": 0, "points": [[1, 1]]}, {"at_ms": 20, "points": [[2, 3]]}],
|
||||
"target_description": "observed control",
|
||||
},
|
||||
}
|
||||
inputs = []
|
||||
|
||||
async def native(command, args, *, timeout=None):
|
||||
if command == "inspect_control":
|
||||
if name == "wait_until":
|
||||
raise asyncio.CancelledError
|
||||
return missing() if args["selector"] == TARGET else observed()
|
||||
inputs.append(command)
|
||||
raise asyncio.CancelledError
|
||||
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await run(name, requests[name], native)
|
||||
assert inputs == (
|
||||
[]
|
||||
if name == "wait_until"
|
||||
else ["touch_gesture" if name == "touch_gesture" else "control_action"]
|
||||
)
|
||||
@@ -0,0 +1,270 @@
|
||||
"""Mixed native sequences exercise production preflight and dispatch over mocks."""
|
||||
|
||||
from copy import deepcopy
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.mcp.action_manifest import DEVICE_ACTIONS
|
||||
from artemis.mcp.actuators import MockActuator
|
||||
|
||||
|
||||
SELECTOR = {"resource_id": "fixture:id/editor"}
|
||||
TOKEN = "a" * 64
|
||||
|
||||
|
||||
def wait_step():
|
||||
return {"action": "wait_until", "selector": SELECTOR, "state": "visible", "timeout_ms": 100}
|
||||
|
||||
|
||||
def control_step():
|
||||
return {
|
||||
"action": "control_action",
|
||||
"request": {"selector": SELECTOR, "action": "set_text", "text": " exact 😀 \n"},
|
||||
}
|
||||
|
||||
|
||||
def touch_step():
|
||||
return {
|
||||
"action": "touch_gesture",
|
||||
"frames": [
|
||||
{"at_ms": 0, "points": [[250, 500]]},
|
||||
{"at_ms": 150, "points": [[250, 500]]},
|
||||
{"at_ms": 400, "points": [[500, 750]]},
|
||||
],
|
||||
"coordinate_space": "normalized",
|
||||
"target_description": "observed canvas",
|
||||
}
|
||||
|
||||
|
||||
def click_step():
|
||||
return {"action": "click", "target": [50, 100], "target_description": "observed button"}
|
||||
|
||||
|
||||
def observed():
|
||||
return {
|
||||
"success": True,
|
||||
"execution": "observed",
|
||||
"code": "OK",
|
||||
"node": {
|
||||
"token": TOKEN,
|
||||
"content_signature": "b" * 64,
|
||||
"visible": True,
|
||||
"enabled": True,
|
||||
"editable": True,
|
||||
"password": False,
|
||||
"text": " A😀B ",
|
||||
"selection_start": 1,
|
||||
"selection_end": 3,
|
||||
"actions": ["set_text", "set_selection"],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def sequence_limit(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"artemis.config.agent.load_agent_config",
|
||||
lambda: SimpleNamespace(
|
||||
pro=SimpleNamespace(execution=SimpleNamespace(max_burst_actions=10))
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def actuator_with_native(capabilities=DEVICE_ACTIONS):
|
||||
actuator = MockActuator(width=720, height=1600, capabilities=capabilities)
|
||||
|
||||
async def native(command, args, *, timeout=None):
|
||||
if command == "inspect_control":
|
||||
return observed()
|
||||
return {
|
||||
"success": True,
|
||||
"execution": "completed",
|
||||
"code": "OK",
|
||||
"native_marker": command,
|
||||
}
|
||||
|
||||
actuator.controller.native_request = AsyncMock(side_effect=native)
|
||||
actuator.controller.tap_at = AsyncMock(return_value=SimpleNamespace(error=None))
|
||||
actuator.controller.swipe_coords = AsyncMock()
|
||||
return actuator
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"bad_member",
|
||||
[
|
||||
{**wait_step(), "timeout_ms": 0},
|
||||
{**wait_step(), "state": "text"},
|
||||
{**wait_step(), "timeout_ms": 3001},
|
||||
{"action": "control_action", "request": {"selector": SELECTOR, "action": "set_progress"}},
|
||||
{
|
||||
"action": "control_action",
|
||||
"request": {"selector": SELECTOR, "action": "set_text", "text": "x" * 1025},
|
||||
},
|
||||
{
|
||||
"action": "control_action",
|
||||
"request": {"selector": SELECTOR, "action": "click", "text": "unexpected"},
|
||||
},
|
||||
{
|
||||
**touch_step(),
|
||||
"frames": [
|
||||
{"at_ms": 0, "points": [[100, 100]]},
|
||||
{"at_ms": 400, "points": [[1001, 100]]},
|
||||
],
|
||||
},
|
||||
{
|
||||
**touch_step(),
|
||||
"frames": [
|
||||
{"at_ms": 0, "points": [[100, 100], [300, 100]]},
|
||||
{"at_ms": 400, "points": [[300, 100], [100, 100]]},
|
||||
],
|
||||
},
|
||||
{
|
||||
**touch_step(),
|
||||
"frames": [
|
||||
{"at_ms": 0, "points": [[100, 100]]},
|
||||
{"at_ms": 400, "points": [[200, 200], [300, 300]]},
|
||||
],
|
||||
},
|
||||
],
|
||||
)
|
||||
async def test_all_native_parameters_and_paths_are_preflighted_before_even_the_first_click(
|
||||
bad_member,
|
||||
):
|
||||
actuator = actuator_with_native()
|
||||
result = await actuator.action_sequence(
|
||||
[click_step(), wait_step(), control_step(), touch_step(), bad_member]
|
||||
)
|
||||
assert not result.ok and result.code.value == "INVALID_ARGS"
|
||||
actuator.controller.native_request.assert_not_awaited()
|
||||
actuator.controller.tap_at.assert_not_awaited()
|
||||
actuator.controller.swipe_coords.assert_not_awaited()
|
||||
assert actuator.action_history == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("missing_capability", ["wait_until", "control_action", "touch_gesture"])
|
||||
async def test_every_native_member_capability_is_checked_before_any_input(missing_capability):
|
||||
actuator = actuator_with_native(DEVICE_ACTIONS - {missing_capability})
|
||||
result = await actuator.action_sequence(
|
||||
[click_step(), wait_step(), control_step(), touch_step()]
|
||||
)
|
||||
assert not result.ok and result.code.value == "UNSUPPORTED"
|
||||
assert result.metadata == {"execution": "not_started", "side_effect": "none"}
|
||||
actuator.controller.native_request.assert_not_awaited()
|
||||
actuator.controller.tap_at.assert_not_awaited()
|
||||
actuator.controller.swipe_coords.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mixed_wait_control_and_touch_keep_native_receipts_and_exact_text_in_order():
|
||||
actuator = actuator_with_native()
|
||||
steps = [wait_step(), control_step(), touch_step()]
|
||||
original = deepcopy(steps)
|
||||
result = await actuator.action_sequence(steps)
|
||||
assert result.ok and result.metadata["side_effect"] == "possible"
|
||||
assert steps == original
|
||||
calls = actuator.controller.native_request.await_args_list
|
||||
assert [call.args[0] for call in calls] == [
|
||||
"inspect_control",
|
||||
"inspect_control",
|
||||
"control_action",
|
||||
"touch_gesture",
|
||||
]
|
||||
assert calls[2].args[1]["text"] == " exact 😀 \n"
|
||||
assert calls[2].args[1]["expected_token"] == TOKEN
|
||||
receipts = result.metadata["sequence_results"]
|
||||
assert [item["action"] for item in receipts] == [
|
||||
"wait_until",
|
||||
"control_action",
|
||||
"touch_gesture",
|
||||
]
|
||||
assert receipts[0]["read_only"] and receipts[0]["side_effect"] == "none"
|
||||
assert receipts[0]["native_execution"] == "observed"
|
||||
assert receipts[0]["node"]["selection_end"] == 3
|
||||
assert receipts[1]["native_receipt"]["native_marker"] == "control_action"
|
||||
assert receipts[2]["native_receipt"]["native_marker"] == "touch_gesture"
|
||||
assert receipts[0]["ended_ms"] <= receipts[1]["started_ms"] <= receipts[2]["started_ms"]
|
||||
actuator.controller.tap_at.assert_not_awaited()
|
||||
actuator.controller.swipe_coords.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("failure", [False, True])
|
||||
async def test_readonly_native_wait_never_counts_as_an_input_side_effect(failure):
|
||||
actuator = actuator_with_native()
|
||||
if failure:
|
||||
actuator.controller.native_request.return_value = None
|
||||
actuator.controller.native_request.side_effect = lambda *args, **kwargs: {
|
||||
"success": False,
|
||||
"execution": "not_started",
|
||||
"code": "UNAVAILABLE",
|
||||
"error": "No active native tree",
|
||||
}
|
||||
result = await actuator.action_sequence([wait_step()])
|
||||
assert result.ok is not failure
|
||||
assert result.metadata["side_effect"] == "none"
|
||||
receipt = result.metadata["sequence_results"][0]
|
||||
assert receipt["read_only"] and receipt["side_effect"] == "none"
|
||||
assert receipt["native_execution"] == ("not_started" if failure else "observed")
|
||||
assert [call.args[0] for call in actuator.controller.native_request.await_args_list] == [
|
||||
"inspect_control"
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("unknown_member", ["control_action", "touch_gesture"])
|
||||
async def test_unknown_native_input_stops_every_remaining_member_and_preserves_original_receipt(
|
||||
unknown_member,
|
||||
):
|
||||
actuator = actuator_with_native()
|
||||
uncertain = {
|
||||
"success": False,
|
||||
"execution": "unknown",
|
||||
"code": "TRANSPORT_ERROR",
|
||||
"error": "Reply lost; do not replay input",
|
||||
"native_marker": "unacknowledged input",
|
||||
}
|
||||
|
||||
async def native(command, args, *, timeout=None):
|
||||
return observed() if command == "inspect_control" else uncertain
|
||||
|
||||
actuator.controller.native_request.side_effect = native
|
||||
member = control_step() if unknown_member == "control_action" else touch_step()
|
||||
result = await actuator.action_sequence([wait_step(), member, click_step(), touch_step()])
|
||||
assert not result.ok and result.metadata["side_effect"] == "possible"
|
||||
receipts = result.metadata["sequence_results"]
|
||||
assert receipts[1]["execution"] == "unknown"
|
||||
assert receipts[1]["native_execution"] == "unknown"
|
||||
assert receipts[1]["native_receipt"] == uncertain
|
||||
assert [item["execution"] for item in receipts[2:]] == ["not_started", "not_started"]
|
||||
writes = [
|
||||
call.args[0]
|
||||
for call in actuator.controller.native_request.await_args_list
|
||||
if call.args[0] != "inspect_control"
|
||||
]
|
||||
assert writes == [unknown_member]
|
||||
actuator.controller.tap_at.assert_not_awaited()
|
||||
actuator.controller.swipe_coords.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_normalized_touch_in_sequence_is_mapped_to_physical_pixels_exactly_once():
|
||||
actuator = actuator_with_native()
|
||||
result = await actuator.action_sequence([touch_step()])
|
||||
assert result.ok
|
||||
command, sent = actuator.controller.native_request.await_args.args
|
||||
assert command == "touch_gesture" and sent["screen_size"] == [720, 1600]
|
||||
expected = [
|
||||
{"at_ms": 0, "points": [[180, 800]]},
|
||||
{"at_ms": 150, "points": [[180, 800]]},
|
||||
{"at_ms": 400, "points": [[360, 1200]]},
|
||||
]
|
||||
assert sent["frames"] == expected
|
||||
receipt = result.metadata["sequence_results"][0]
|
||||
assert receipt["plan"]["frames"] == expected
|
||||
assert receipt["plan"]["coordinate_space"] == "pixel"
|
||||
assert receipt["args"]["coordinate_space"] == "normalized"
|
||||
assert receipt["args"]["frames"] == touch_step()["frames"]
|
||||
@@ -25,6 +25,34 @@ def delta(arguments, *, name=None, index=0):
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"name", ["pinch", "touch_gesture", "read_control", "wait_until", "scroll_to"]
|
||||
)
|
||||
def test_native_small_actions_stop_argument_inflation(name):
|
||||
guard = ToolArgumentGuard()
|
||||
guard.observe(
|
||||
SimpleNamespace(
|
||||
tool_call_chunks=[{"index": 0, "name": name, "args": "x" * SMALL_ACTION_ARGUMENT_LIMIT}]
|
||||
)
|
||||
)
|
||||
with pytest.raises(ToolArgumentOverflow) as error:
|
||||
guard.observe(SimpleNamespace(tool_call_chunks=[{"index": 0, "args": "x"}]))
|
||||
assert error.value.details["tool"] == name
|
||||
assert error.value.details["limit_chars"] == SMALL_ACTION_ARGUMENT_LIMIT
|
||||
|
||||
|
||||
def test_standalone_native_text_allows_valid_escaped_text_but_remains_bounded():
|
||||
guard = ToolArgumentGuard()
|
||||
escaped = json.dumps({"text": "中" * 65536})
|
||||
assert len(escaped) > SMALL_ACTION_ARGUMENT_LIMIT
|
||||
guard.observe(
|
||||
SimpleNamespace(tool_call_chunks=[{"index": 0, "name": "control_action", "args": escaped}])
|
||||
)
|
||||
with pytest.raises(ToolArgumentOverflow) as error:
|
||||
guard.observe(SimpleNamespace(tool_call_chunks=[{"index": 0, "args": "x" * (512 * 1024)}]))
|
||||
assert error.value.details["limit_chars"] == 512 * 1024
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def isolate(monkeypatch, tmp_path):
|
||||
llm._ENDPOINT_BREAKER._states.clear()
|
||||
|
||||
@@ -0,0 +1,209 @@
|
||||
"""Hierarchy native semantics must survive both supported observation encodings."""
|
||||
|
||||
from copy import deepcopy
|
||||
import json
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.clients.accessibility_client import normalize_helper_elements
|
||||
from artemis.clients.ui_automator_client import _parse_hierarchy_xml_to_elements
|
||||
from artemis.utils.native_control_state import informative_native_control, native_control_state
|
||||
from artemis.utils.visualization import format_minimal_list_with_elements
|
||||
|
||||
|
||||
RANGE = {"min": 0, "max": 100, "current": 37.5, "type": 1}
|
||||
ACTIONS = ["set_progress", "set_selection", "set_text"]
|
||||
|
||||
|
||||
def hierarchy_node(**overrides):
|
||||
return {
|
||||
"text": "",
|
||||
"content-desc": "",
|
||||
"class": "android.widget.SeekBar",
|
||||
"package": "com.artemis.fixture",
|
||||
"resource-id": "com.artemis.fixture:id/range",
|
||||
"bounds": "[10,20][210,80]",
|
||||
"enabled": True,
|
||||
"editable": False,
|
||||
"password": False,
|
||||
"checked": False,
|
||||
"checkable": False,
|
||||
"selected": False,
|
||||
"focused": False,
|
||||
"scrollable": False,
|
||||
"visible-to-user": True,
|
||||
"selection_start": -1,
|
||||
"selection_end": -1,
|
||||
"range": deepcopy(RANGE),
|
||||
"actions": list(ACTIONS),
|
||||
**overrides,
|
||||
}
|
||||
|
||||
|
||||
def encode_node(node, encoding):
|
||||
if encoding == "json":
|
||||
return normalize_helper_elements(json.loads(json.dumps([node])))[0]
|
||||
attributes = {}
|
||||
for key, value in node.items():
|
||||
name = key.replace("_", "-") if key in {"selection_start", "selection_end"} else key
|
||||
attributes[name] = (
|
||||
json.dumps(value, ensure_ascii=False)
|
||||
if isinstance(value, (dict, list, bool)) or value is None
|
||||
else str(value)
|
||||
)
|
||||
root = ET.Element("hierarchy")
|
||||
ET.SubElement(root, "node", attributes)
|
||||
return _parse_hierarchy_xml_to_elements(ET.tostring(root, encoding="unicode"))[0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("encoding", ["xml", "json"])
|
||||
def test_native_range_actions_flags_and_utf16_selection_map_equally_from_xml_and_json(encoding):
|
||||
state = native_control_state(
|
||||
encode_node(hierarchy_node(editable=True, selection_start=1, selection_end=3), encoding)
|
||||
)
|
||||
assert state == {
|
||||
"range": RANGE,
|
||||
"actions": ACTIONS,
|
||||
"enabled": True,
|
||||
"scrollable": False,
|
||||
"editable": True,
|
||||
"password": False,
|
||||
"checked": False,
|
||||
"checkable": False,
|
||||
"selected": False,
|
||||
"focused": False,
|
||||
"selection_start": 1,
|
||||
"selection_end": 3,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("encoding", ["xml", "json"])
|
||||
@pytest.mark.parametrize(
|
||||
"semantics",
|
||||
[
|
||||
{"range": RANGE, "actions": []},
|
||||
{"range": None, "actions": ["set_progress"]},
|
||||
{"range": None, "actions": ["click"]},
|
||||
{"range": None, "actions": [], "scrollable": True},
|
||||
],
|
||||
)
|
||||
def test_unnamed_native_controls_are_discoverable_in_the_actual_visualization(encoding, semantics):
|
||||
node = encode_node(hierarchy_node(**semantics), encoding)
|
||||
text, elements, labels = format_minimal_list_with_elements([node], 400, 800)
|
||||
assert len(elements) == 1 and labels == ["1"]
|
||||
element = elements[0]
|
||||
assert element["text"] == "(native control)"
|
||||
assert element["native_text"] == "" and element["content_desc"] == ""
|
||||
assert element["resource_id"] == "com.artemis.fixture:id/range"
|
||||
assert element["package_name"] == "com.artemis.fixture"
|
||||
assert element["class"] == "android.widget.SeekBar"
|
||||
assert element["center"] == [110, 50] and element["bounds"] == [10, 20, 210, 80]
|
||||
assert "Control: '(native control)'" in text
|
||||
assert 'package_name: "com.artemis.fixture"' in text
|
||||
if semantics["range"] is not None:
|
||||
assert element["range"] == RANGE and '"current": 37.5' in text
|
||||
if semantics["actions"]:
|
||||
assert element["actions"] == semantics["actions"]
|
||||
assert "actions: " + ",".join(semantics["actions"]) in text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("encoding", ["xml", "json"])
|
||||
def test_native_text_description_package_and_selection_preserve_exact_observed_data(encoding):
|
||||
raw = hierarchy_node(
|
||||
text=" A😀B \n\t",
|
||||
**{
|
||||
"content-desc": " description \n\t",
|
||||
"class": "android.widget.EditText",
|
||||
"package": "com.artemis.fixture.editor",
|
||||
"editable": True,
|
||||
"selection_start": 1,
|
||||
"selection_end": 3,
|
||||
},
|
||||
)
|
||||
node = encode_node(raw, encoding)
|
||||
original = deepcopy(node)
|
||||
text, elements, _ = format_minimal_list_with_elements([node], 400, 800)
|
||||
element = elements[0]
|
||||
assert element["native_text"] == raw["text"]
|
||||
assert element["input_value"] == raw["text"]
|
||||
assert element["content_desc"] == raw["content-desc"]
|
||||
assert element["package_name"] == raw["package"]
|
||||
assert (element["selection_start"], element["selection_end"]) == (1, 3)
|
||||
assert element["range"] == RANGE and element["actions"] == ACTIONS
|
||||
assert json.dumps(raw["content-desc"], ensure_ascii=False) in text
|
||||
assert node == original
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"bad_range",
|
||||
[
|
||||
None,
|
||||
[],
|
||||
"invalid json",
|
||||
"{}",
|
||||
{"min": 0, "max": 100},
|
||||
{"min": 10, "max": 0, "current": 5},
|
||||
{"min": 0, "max": 10, "current": 11},
|
||||
{"min": False, "max": 10, "current": 1},
|
||||
{"min": "0", "max": 10, "current": 1},
|
||||
{"min": 0, "max": float("inf"), "current": 1},
|
||||
{"min": 0, "max": 10, "current": float("nan")},
|
||||
],
|
||||
)
|
||||
def test_malformed_range_does_not_hide_other_valid_native_metadata(bad_range):
|
||||
assert native_control_state({"range": bad_range, "actions": ["click"], "enabled": "true"}) == {
|
||||
"actions": ["click"],
|
||||
"enabled": True,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("encoding", ["xml", "json"])
|
||||
@pytest.mark.parametrize("field", ["min", "max", "current"])
|
||||
def test_huge_range_integers_are_omitted_without_crashing_observation(encoding, field):
|
||||
value = {**RANGE, field: 10**1000}
|
||||
node = encode_node(hierarchy_node(range=value), encoding)
|
||||
state = native_control_state(node)
|
||||
assert "range" not in state and state["actions"] == ACTIONS
|
||||
_, elements, _ = format_minimal_list_with_elements([node], 400, 800)
|
||||
assert len(elements) == 1 and "range" not in elements[0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"key", ["selection_start", "selection-start", "selection_end", "selection-end"]
|
||||
)
|
||||
@pytest.mark.parametrize("bad_offset", [True, 1.5, "1.5", "--1", "²", -2, 10**1000, "9" * 100])
|
||||
def test_invalid_or_impossible_selection_offsets_are_ignored_without_raising(key, bad_offset):
|
||||
state = native_control_state({key: bad_offset, "enabled": True})
|
||||
assert state == {"enabled": True}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", [-1, 0, 3, 2**31 - 1, "-1", "0", "3", str(2**31 - 1)])
|
||||
def test_supported_native_int_selection_offsets_are_kept(value):
|
||||
assert native_control_state({"selection-start": value}) == {"selection_start": int(value)}
|
||||
|
||||
|
||||
def test_encoded_metadata_is_bounded_and_rejects_nonstring_action_names():
|
||||
state = native_control_state(
|
||||
{
|
||||
"range": " " * 8193,
|
||||
"actions": ["click", None, 7, False, "x" * 65, *["set_progress"] * 40],
|
||||
}
|
||||
)
|
||||
assert "range" not in state
|
||||
assert state["actions"] == ["click", *["set_progress"] * 27]
|
||||
assert "actions" not in native_control_state({"actions": " " * 8193})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", [None, 0, 1, "TRUE", "False", "yes", {}, []])
|
||||
def test_metadata_flags_require_actual_boolean_values_or_canonical_xml_strings(value):
|
||||
assert native_control_state({"enabled": value}) == {}
|
||||
|
||||
|
||||
def test_passive_empty_layouts_do_not_become_actionable_controls():
|
||||
state = native_control_state({"actions": ["focus", "show_on_screen"], "range": None})
|
||||
assert not informative_native_control(state)
|
||||
text, elements, labels = format_minimal_list_with_elements(
|
||||
[hierarchy_node(actions=[], range=None)], 400, 800
|
||||
)
|
||||
assert (text, elements, labels) == ("", [], [])
|
||||
Reference in new issue
Block a user