feat(android): add state conditions and measured gesture feedback
Add optional ensure_state, gesture region and final value feedback shared by Flash and Pro. Preserve completed inputs when verification fails and stop repeated ineffective strategies. Regenerate pinned tool schemas for the new optional arguments.
This commit is contained in:
chenmingxuan committed
1 parent
6a69a595ef
commit
4ea7939d9c
20 files changed
+2779
-33
No files matched your search
@@ -0,0 +1,346 @@
|
||||
"""Regress real actuator dispatch, state verification and value convergence without devices."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.mcp import value_feedback
|
||||
from artemis.mcp.action_manifest import DEVICE_ACTIONS
|
||||
from artemis.mcp.actuators import MockActuator
|
||||
from artemis.mcp.gestures import DragRequest, plan_drag
|
||||
from artemis.mcp.native_actions import PinchRequest, plan_pinch
|
||||
from artemis.mcp.value_feedback import read_value
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def deterministic_clock(monkeypatch):
|
||||
now = [0.0]
|
||||
|
||||
async def sleep(seconds):
|
||||
now[0] += seconds
|
||||
|
||||
monkeypatch.setattr(value_feedback, "monotonic", lambda: now[0])
|
||||
monkeypatch.setattr(value_feedback, "sleep", sleep)
|
||||
monkeypatch.setattr(
|
||||
"artemis.config.agent.load_agent_config",
|
||||
lambda: SimpleNamespace(
|
||||
pro=SimpleNamespace(execution=SimpleNamespace(max_burst_actions=10))
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def setup_actuator(*, current="100", after="100", readable=True):
|
||||
actuator = MockActuator(width=720, height=1600, capabilities=DEVICE_ACTIONS)
|
||||
state = {"text": current, "playing": False}
|
||||
events = []
|
||||
|
||||
async def native(command, args, **kwargs):
|
||||
events.append(command)
|
||||
if command == "control_action":
|
||||
state["playing"] = not state["playing"]
|
||||
return {"success": True, "execution": "completed", "code": "OK"}
|
||||
is_player = args["selector"]["resource_id"] == "player"
|
||||
return {
|
||||
"success": True,
|
||||
"execution": "observed",
|
||||
"code": "OK",
|
||||
"node": {
|
||||
"token": "a" * 64,
|
||||
"content_signature": "b" * 64,
|
||||
"visible": True,
|
||||
"enabled": True,
|
||||
"password": False,
|
||||
"actions": ["click"],
|
||||
"text": state["text"] if readable else None,
|
||||
"content_desc": ("暂停" if state["playing"] else "播放")
|
||||
if is_player
|
||||
else "readout",
|
||||
},
|
||||
}
|
||||
|
||||
async def swipe(*args):
|
||||
events.append("swipe")
|
||||
state.update(text=after, playing=True)
|
||||
return None
|
||||
|
||||
actuator.controller.native_request = AsyncMock(side_effect=native)
|
||||
actuator.controller.swipe_coords = AsyncMock(side_effect=swipe)
|
||||
actuator.controller.tap_at = AsyncMock(return_value=SimpleNamespace(error=None))
|
||||
return actuator, state, events
|
||||
|
||||
|
||||
def feedback(**overrides):
|
||||
return {
|
||||
"selector": {"resource_id": "readout"},
|
||||
"value_format": "number",
|
||||
"target_value": 90.0,
|
||||
"tolerance": 1.0,
|
||||
"stable_ms": 100,
|
||||
"timeout_ms": 500,
|
||||
**overrides,
|
||||
}
|
||||
|
||||
|
||||
def drag(**overrides):
|
||||
return {
|
||||
"start": [300, 1050],
|
||||
"end": [325, 1050],
|
||||
"coordinate_space": "pixel",
|
||||
"target_description": "observed timeline",
|
||||
**overrides,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_two_ineffective_attempts_block_third_but_allow_meaningful_strategy_change():
|
||||
actuator, _, _ = setup_actuator()
|
||||
for _ in range(2):
|
||||
result = await actuator.drag(**drag(feedback=feedback()))
|
||||
assert not result.ok and result.metadata["execution"] == "completed"
|
||||
assert result.metadata["value_feedback"]["after_value"] == 100
|
||||
result = await actuator.drag(**drag(feedback=feedback()))
|
||||
assert not result.ok and "NO_PROGRESS" in result.message
|
||||
assert result.metadata["execution"] == "not_started"
|
||||
assert actuator.controller.swipe_coords.await_count == 2
|
||||
# Tiny endpoint jitter is the same strategy, not an escape from the guard.
|
||||
await actuator.drag(**drag(end=[328, 1050], feedback=feedback()))
|
||||
assert actuator.controller.swipe_coords.await_count == 2
|
||||
await actuator.drag(**drag(end=[424, 1050], feedback=feedback()))
|
||||
assert actuator.controller.swipe_coords.await_count == 3
|
||||
# History belongs to this actuator/task, not a global cache.
|
||||
other, _, _ = setup_actuator()
|
||||
await other.drag(**drag(feedback=feedback()))
|
||||
assert other.controller.swipe_coords.await_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_changed_actual_value_resets_stalled_history():
|
||||
actuator, state, _ = setup_actuator()
|
||||
for _ in range(2):
|
||||
await actuator.drag(**drag(feedback=feedback()))
|
||||
state["text"] = "95"
|
||||
await actuator.drag(**drag(feedback=feedback()))
|
||||
assert actuator.controller.swipe_coords.await_count == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_one_pixel_change_across_power_of_two_is_still_the_same_strategy():
|
||||
actuator, _, _ = setup_actuator()
|
||||
for endpoint in (331, 332, 333):
|
||||
await actuator.drag(**drag(end=[endpoint, 1050], feedback=feedback()))
|
||||
assert actuator.controller.swipe_coords.await_count == 2
|
||||
# A substantially slower gesture is a meaningful new approach.
|
||||
await actuator.drag(**drag(end=[333, 1050], duration_ms=1500, feedback=feedback()))
|
||||
assert actuator.controller.swipe_coords.await_count == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sequence_reads_final_value_only_after_pause_and_skips_redundant_toggle():
|
||||
actuator, state, events = setup_actuator(
|
||||
current="11-02 周一 17:43:12", after="11-02 周一 17:42:17"
|
||||
)
|
||||
pause = {
|
||||
"action": "control_action",
|
||||
"request": {
|
||||
"action": "click",
|
||||
"selector": {"resource_id": "player"},
|
||||
"ensure_state": {"state": "content_desc", "expected_text": "播放"},
|
||||
},
|
||||
}
|
||||
result = await actuator.action_sequence(
|
||||
[pause, {"action": "drag", **drag()}, pause],
|
||||
feedback=feedback(
|
||||
value_format="clock", target_value=17.0 * 3600 + 42 * 60 + 17, tolerance=5.0
|
||||
),
|
||||
)
|
||||
assert result.ok and not state["playing"]
|
||||
assert result.metadata["skipped_steps"] == 1
|
||||
assert result.metadata["value_feedback"]["residual"] == 0
|
||||
observation = result.metadata["value_feedback"]["after_observation"]
|
||||
assert observation["node"]["text"] == "11-02 周一 17:42:17"
|
||||
assert observation["stable_for_ms"] >= 100 and observation["samples"] >= 2
|
||||
assert result.metadata["verification"] == "value_matched"
|
||||
assert events.count("control_action") == 1
|
||||
assert events.index("swipe") < events.index("control_action")
|
||||
assert events[-1] == "inspect_control"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_value_refuses_input_and_postwrite_read_failure_preserves_receipt():
|
||||
actuator, _, _ = setup_actuator(readable=False)
|
||||
result = await actuator.drag(**drag(feedback=feedback()))
|
||||
assert not result.ok and result.metadata["execution"] == "not_started"
|
||||
actuator.controller.swipe_coords.assert_not_awaited()
|
||||
actuator, _, _ = setup_actuator(after="unreadable")
|
||||
result = await actuator.drag(**drag(feedback=feedback()))
|
||||
assert not result.ok and result.metadata["execution"] == "completed"
|
||||
assert result.metadata["coordinates"] == [300, 1050, 325, 1050]
|
||||
assert result.metadata["verification"] == "unavailable"
|
||||
actuator.controller.swipe_coords.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_native_feedback_unavailable_has_no_coordinate_fallback():
|
||||
actuator = MockActuator(capabilities=DEVICE_ACTIONS - {"read_control", "wait_until"})
|
||||
actuator.controller.swipe_coords = AsyncMock()
|
||||
result = await actuator.drag(**drag(feedback=feedback()))
|
||||
assert not result.ok and result.code == "UNSUPPORTED"
|
||||
actuator.controller.swipe_coords.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_whole_sequence_rejects_bad_region_before_any_click():
|
||||
actuator, _, _ = setup_actuator()
|
||||
result = await actuator.action_sequence(
|
||||
[
|
||||
{"action": "click", "target": [50, 50], "target_description": "opener"},
|
||||
{"action": "drag", **drag(region=[400, 970, 650, 1118])},
|
||||
]
|
||||
)
|
||||
assert not result.ok
|
||||
actuator.controller.tap_at.assert_not_awaited()
|
||||
actuator.controller.swipe_coords.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failed_state_verification_keeps_completed_members_and_stops_next_input():
|
||||
actuator, _, _ = setup_actuator()
|
||||
|
||||
async def unchanged(command, args, **kwargs):
|
||||
if command == "control_action":
|
||||
return {"success": True, "execution": "completed", "code": "OK"}
|
||||
return {
|
||||
"success": True,
|
||||
"execution": "observed",
|
||||
"code": "OK",
|
||||
"node": {
|
||||
"token": "a" * 64,
|
||||
"visible": True,
|
||||
"enabled": True,
|
||||
"text": "100",
|
||||
"content_desc": "暂停",
|
||||
"actions": ["click"],
|
||||
},
|
||||
}
|
||||
|
||||
actuator.controller.native_request = AsyncMock(side_effect=unchanged)
|
||||
result = await actuator.action_sequence(
|
||||
[
|
||||
{"action": "drag", **drag()},
|
||||
{
|
||||
"action": "control_action",
|
||||
"request": {
|
||||
"action": "click",
|
||||
"selector": {"resource_id": "player"},
|
||||
"ensure_state": {"state": "content_desc", "expected_text": "播放"},
|
||||
"verify_timeout_ms": 1,
|
||||
},
|
||||
},
|
||||
{"action": "click", "target": [50, 50], "target_description": "must not execute"},
|
||||
],
|
||||
feedback=feedback(),
|
||||
)
|
||||
assert not result.ok and result.metadata["execution"] == "partial"
|
||||
assert [r["execution"] for r in result.metadata["sequence_results"]] == [
|
||||
"completed",
|
||||
"completed",
|
||||
"not_started",
|
||||
]
|
||||
assert result.metadata["sequence_results"][1]["verification"] == "condition_not_met"
|
||||
actuator.controller.tap_at.assert_not_awaited()
|
||||
actuator.controller.swipe_coords.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transient_post_action_read_failure_can_settle_without_replaying_input():
|
||||
actuator, _, events = setup_actuator(after="90")
|
||||
original = actuator.controller.native_request.side_effect
|
||||
failed = False
|
||||
|
||||
async def transient(command, args, **kwargs):
|
||||
nonlocal failed
|
||||
if "swipe" in events and not failed:
|
||||
failed = True
|
||||
return {"success": False, "execution": "not_started", "code": "READ_FAILED"}
|
||||
return await original(command, args, **kwargs)
|
||||
|
||||
actuator.controller.native_request = AsyncMock(side_effect=transient)
|
||||
result = await actuator.drag(**drag(feedback=feedback()))
|
||||
assert result.ok and result.metadata["value_feedback"]["after_value"] == 90
|
||||
actuator.controller.swipe_coords.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_feedback_timeout_never_repeats_a_completed_input(monkeypatch):
|
||||
actuator, state, _ = setup_actuator()
|
||||
original = value_feedback.sleep
|
||||
|
||||
async def moving(seconds):
|
||||
state["text"] = str(float(state["text"]) + 1)
|
||||
await original(seconds)
|
||||
|
||||
monkeypatch.setattr(value_feedback, "sleep", moving)
|
||||
result = await actuator.drag(**drag(feedback=feedback()))
|
||||
assert not result.ok and result.code == "TIMEOUT"
|
||||
assert result.metadata["execution"] == "completed"
|
||||
assert result.metadata["verification"] == "unavailable"
|
||||
actuator.controller.swipe_coords.assert_awaited_once()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text", ["17:42:17 / 17:43:12", "25:42:17", "117:42:17", "17:42:170", "17:42"]
|
||||
)
|
||||
def test_clock_readout_rejects_ambiguous_or_invalid_times(text):
|
||||
with pytest.raises(ValueError):
|
||||
read_value({"visible": True, "text": text}, "clock")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"value_format,node",
|
||||
[
|
||||
("number", {"text": "0"}),
|
||||
("clock", {"text": "00:00:00"}),
|
||||
("range", {"range": {"min": 0, "current": 0, "max": 10}}),
|
||||
],
|
||||
)
|
||||
def test_zero_is_an_actual_value(value_format, node):
|
||||
assert read_value({"visible": True, **node}, value_format) == 0
|
||||
|
||||
|
||||
def test_precision_guard_uses_observed_minimum_and_reports_geometric_budget():
|
||||
calibration = {
|
||||
"axis": "x",
|
||||
"mode": "thumb",
|
||||
"anchors": [{"position": 100.0, "value": 0.0}, {"position": 200.0, "value": 1000.0}],
|
||||
"target_value": 125.0,
|
||||
"tolerance": 1.0,
|
||||
}
|
||||
request = {
|
||||
"start": [100, 1050],
|
||||
"coordinate_space": "pixel",
|
||||
"target_description": "observed scale",
|
||||
"calibration": calibration,
|
||||
}
|
||||
with pytest.raises(ValueError, match="rounding"):
|
||||
plan_drag(DragRequest.model_validate(request), 720, 1600)
|
||||
calibration.update(tolerance=5.0, min_movement_px=20.0)
|
||||
with pytest.raises(ValueError, match="minimum effective"):
|
||||
plan_drag(DragRequest.model_validate(request), 720, 1600)
|
||||
calibration.update(min_movement_px=10.0)
|
||||
plan = plan_drag(DragRequest.model_validate(request), 720, 1600)
|
||||
assert plan["calibration"]["pixel_error_budget"] == 0.5
|
||||
assert plan["calibration"]["rounding_error_value"] == 5
|
||||
|
||||
|
||||
def test_both_pinch_fingers_must_stay_in_observed_surface():
|
||||
request = PinchRequest(
|
||||
center=[360, 656],
|
||||
start_span=40,
|
||||
end_span=80,
|
||||
coordinate_space="pixel",
|
||||
target_description="observed timeline",
|
||||
region=[200, 970, 650, 1118],
|
||||
)
|
||||
with pytest.raises(ValueError, match="region"):
|
||||
plan_pinch(request, 720, 1600)
|
||||
Reference in new issue
Block a user