feat: add gesture sequences and calibrated pixel drags
This commit is contained in:
chenmingxuan committed
1 parent
2a3bbaa3fb
commit
479a185192
31 files changed
+2440
-44
No files matched your search
@@ -83,6 +83,8 @@ def test_tool_declarations_match_fixture():
|
||||
"LONG_PRESS_TOOL": "long_press",
|
||||
"INPUT_TEXT_TOOL": "input_text",
|
||||
"SWIPE_TOOL": "swipe",
|
||||
"DRAG_TOOL": "drag",
|
||||
"ACTION_SEQUENCE_TOOL": "action_sequence",
|
||||
"PRESS_KEY_TOOL": "press_key",
|
||||
"MANAGE_APP_TOOL": "manage_app",
|
||||
"WAIT_FOR_DELAY_TOOL": "wait_for_delay",
|
||||
@@ -121,6 +123,8 @@ def test_validator_declaration_order_is_stable():
|
||||
"long_press",
|
||||
"input_text",
|
||||
"swipe",
|
||||
"drag",
|
||||
"action_sequence",
|
||||
"press_key",
|
||||
"read_note",
|
||||
"list_notes",
|
||||
@@ -159,7 +163,7 @@ def test_flash_declarations_are_the_operator_shells_projected():
|
||||
decl_props = declaration.parameters["properties"]
|
||||
assert list(decl_props) == list(shell_props), spec.name
|
||||
for param, schema in shell_props.items():
|
||||
assert decl_props[param] == _without_optional_noise(schema), (
|
||||
assert _without_optional_noise(decl_props[param]) == _without_optional_noise(schema), (
|
||||
f"'{spec.name}.{param}' diverges between the operator shell and the"
|
||||
" Flash declaration."
|
||||
)
|
||||
|
||||
@@ -0,0 +1,400 @@
|
||||
"""Behavioral regressions for short-lived controls and measured scale movement."""
|
||||
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from artemis.mcp.action_server import build_action_server
|
||||
from artemis.mcp.action_session import ActionSession
|
||||
from artemis.mcp.actuators import MockActuator
|
||||
from artemis.mcp.gestures import DragRequest, plan_drag
|
||||
|
||||
|
||||
def test_pan_uses_decreasing_observed_time_scale_without_overshoot():
|
||||
request = DragRequest(
|
||||
start=[190, 1059],
|
||||
coordinate_space="pixel",
|
||||
target_description="timeline",
|
||||
calibration={
|
||||
"axis": "y",
|
||||
"mode": "pan",
|
||||
"anchors": [
|
||||
{"position": 1075, "value": 38400},
|
||||
{"position": 1187, "value": 37800},
|
||||
],
|
||||
"target_value": 37800,
|
||||
"indicator_position": 1215,
|
||||
},
|
||||
)
|
||||
planned = plan_drag(request, 720, 1600)
|
||||
assert planned["coordinates"] == [190, 1059, 190, 1087]
|
||||
assert planned["calibration"]["value_per_pixel"] == pytest.approx(-600 / 112)
|
||||
assert planned["calibration"]["verification"] == "pending_observation"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"axis,start,expected", [("x", [80, 200], [125, 200]), ("y", [200, 80], [200, 125])]
|
||||
)
|
||||
def test_thumb_maps_value_to_the_requested_axis(axis, start, expected):
|
||||
planned = plan_drag(
|
||||
DragRequest(
|
||||
start=start,
|
||||
coordinate_space="pixel",
|
||||
target_description="slider",
|
||||
calibration={
|
||||
"axis": axis,
|
||||
"mode": "thumb",
|
||||
"anchors": [
|
||||
{"position": 100, "value": 0},
|
||||
{"position": 200, "value": 100},
|
||||
],
|
||||
"target_value": 25,
|
||||
},
|
||||
),
|
||||
720,
|
||||
1600,
|
||||
)
|
||||
assert planned["coordinates"] == [*start, *expected]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pixel_drag_is_not_quantized_through_normalized_coordinates():
|
||||
actuator = MockActuator(width=720, height=1600)
|
||||
session = await ActionSession(build_action_server(actuator)).start()
|
||||
try:
|
||||
result = await session.call(
|
||||
"drag",
|
||||
{
|
||||
"start": [191, 1001],
|
||||
"end": [191, 1032],
|
||||
"coordinate_space": "pixel",
|
||||
"target_description": "timeline",
|
||||
},
|
||||
)
|
||||
assert result.ok
|
||||
assert result.metadata["coordinates"] == [191, 1001, 191, 1032]
|
||||
assert actuator.action_history == [
|
||||
{
|
||||
"action": "swipe",
|
||||
"start": (191, 1001),
|
||||
"end": (191, 1032),
|
||||
"duration_ms": 300,
|
||||
}
|
||||
]
|
||||
finally:
|
||||
await session.aclose()
|
||||
|
||||
|
||||
def steps():
|
||||
return [
|
||||
{"action": "click", "target": [200, 900], "target_description": "open scale"},
|
||||
{"action": "wait", "time_in_ms": 1},
|
||||
{
|
||||
"action": "drag",
|
||||
"start": [190, 1001],
|
||||
"end": [190, 1032],
|
||||
"coordinate_space": "pixel",
|
||||
"target_description": "scale handle",
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sequence_has_no_observation_between_inputs_and_returns_member_receipts():
|
||||
actuator = MockActuator(width=720, height=1600)
|
||||
actuator.controller.tap_at = AsyncMock(return_value=MagicMock(error=None))
|
||||
actuator.controller.swipe_coords = AsyncMock(return_value=None)
|
||||
actuator.take_screenshot = AsyncMock()
|
||||
result = await actuator.action_sequence(steps())
|
||||
assert result.ok
|
||||
actuator.controller.tap_at.assert_awaited_once_with(144, 1440)
|
||||
actuator.controller.swipe_coords.assert_awaited_once_with(190, 1001, 190, 1032, 300)
|
||||
actuator.take_screenshot.assert_not_awaited()
|
||||
receipts = result.metadata["sequence_results"]
|
||||
assert [r["execution"] for r in receipts] == ["completed"] * 3
|
||||
assert receipts[0]["ended_ms"] <= receipts[1]["started_ms"] <= receipts[2]["started_ms"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_late_invalid_member_prevents_even_the_first_input():
|
||||
actuator = MockActuator(width=720, height=1600)
|
||||
actuator.controller.tap_at = AsyncMock()
|
||||
invalid = steps()
|
||||
invalid[-1]["end"] = [720, 1032]
|
||||
result = await actuator.action_sequence(invalid)
|
||||
assert not result.ok and result.code.value == "INVALID_ARGS"
|
||||
actuator.controller.tap_at.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failure_keeps_completed_members_and_does_not_dispatch_the_remainder():
|
||||
actuator = MockActuator(width=720, height=1600)
|
||||
actuator.controller.tap_at = AsyncMock(
|
||||
side_effect=[MagicMock(error=None), RuntimeError("transport lost")]
|
||||
)
|
||||
member = {"action": "click", "target": [200, 900], "target_description": "control"}
|
||||
result = await actuator.action_sequence([member, member, member])
|
||||
assert not result.ok
|
||||
assert actuator.controller.tap_at.await_count == 2
|
||||
assert [r["execution"] for r in result.metadata["sequence_results"]] == [
|
||||
"completed",
|
||||
"unknown",
|
||||
"not_started",
|
||||
]
|
||||
assert result.metadata["execution"] == "partial"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancellation_is_not_swallowed_or_retried():
|
||||
actuator = MockActuator(width=720, height=1600)
|
||||
actuator.ctx.data_engine = MagicMock(current_step_id="step")
|
||||
actuator.controller.swipe_coords = AsyncMock(side_effect=asyncio.CancelledError)
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await actuator.action_sequence(steps())
|
||||
assert len(actuator.action_history) == 1
|
||||
assert actuator.controller.swipe_coords.await_count == 1
|
||||
receipts = [
|
||||
c.kwargs["payload"]["receipt"]
|
||||
for c in actuator.ctx.data_engine.record_trace.call_args_list
|
||||
if "receipt" in c.kwargs["payload"]
|
||||
]
|
||||
assert [r["execution"] for r in receipts] == ["completed", "completed", "unknown"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pause_at_member_boundary_does_not_dispatch_later_inputs(tmp_path):
|
||||
from artemis.runtime.task_control import TaskControl
|
||||
|
||||
actuator = MockActuator(width=720, height=1600)
|
||||
control = object.__new__(TaskControl)
|
||||
control.directory = tmp_path
|
||||
actuator.ctx.task_control = control
|
||||
|
||||
async def tap(*_):
|
||||
(tmp_path / "pause.request").write_text("{}")
|
||||
return SimpleNamespace(error=None)
|
||||
|
||||
actuator.controller.tap_at = AsyncMock(side_effect=tap)
|
||||
actuator.controller.swipe_coords = AsyncMock()
|
||||
result = await actuator.action_sequence(steps())
|
||||
assert not result.ok
|
||||
assert result.metadata["completed_steps"] == 1
|
||||
assert result.metadata["sequence_results"][1]["execution"] == "not_started"
|
||||
actuator.controller.swipe_coords.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_over_limit_is_rejected_before_dispatch(monkeypatch):
|
||||
monkeypatch.setattr(
|
||||
"artemis.config.agent.load_agent_config",
|
||||
lambda: SimpleNamespace(
|
||||
pro=SimpleNamespace(execution=SimpleNamespace(max_burst_actions=2)),
|
||||
),
|
||||
)
|
||||
actuator = MockActuator(width=720, height=1600)
|
||||
result = await actuator.action_sequence(steps())
|
||||
assert not result.ok and "limit of 2" in result.message
|
||||
assert actuator.action_history == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("values", [(0, 0), (1e308, -1e308), (float("nan"), 1), (True, 5)])
|
||||
def test_degenerate_and_nonfinite_calibration_is_rejected(values):
|
||||
with pytest.raises(ValueError):
|
||||
request = DragRequest(
|
||||
start=[100, 100],
|
||||
coordinate_space="pixel",
|
||||
target_description="timeline",
|
||||
calibration={
|
||||
"axis": "y",
|
||||
"mode": "pan",
|
||||
"indicator_position": 100,
|
||||
"anchors": [
|
||||
{"position": 100, "value": values[0]},
|
||||
{"position": 200, "value": values[1]},
|
||||
],
|
||||
"target_value": 3,
|
||||
},
|
||||
)
|
||||
plan_drag(request, 720, 1600)
|
||||
|
||||
|
||||
def test_flash_and_pro_schemas_preserve_null_for_calculated_endpoints():
|
||||
from langchain_core.utils.function_calling import convert_to_openai_tool
|
||||
from artemis.mcp.action_specs import tool_declaration, operator_shell_tool
|
||||
|
||||
flash = tool_declaration("drag").parameters
|
||||
pro = convert_to_openai_tool(operator_shell_tool("drag"))["function"]["parameters"]
|
||||
assert flash == pro
|
||||
assert {"type": "null"} in flash["properties"]["end"]["anyOf"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_conflicting_endpoint_has_actionable_error_and_no_input():
|
||||
actuator = MockActuator(width=720, height=1600)
|
||||
result = await actuator.drag(
|
||||
start=[190, 1200],
|
||||
end=[190, 1100],
|
||||
coordinate_space="pixel",
|
||||
target_description="timeline",
|
||||
calibration={
|
||||
"axis": "y",
|
||||
"mode": "pan",
|
||||
"anchors": [{"position": 1000, "value": 60}, {"position": 1100, "value": 0}],
|
||||
"target_value": 30,
|
||||
"indicator_position": 1050,
|
||||
},
|
||||
)
|
||||
assert not result.ok and "set end=null" in result.message
|
||||
assert actuator.action_history == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("mode", ["flash", "pro"])
|
||||
async def test_each_mode_exposes_the_receipt_to_its_next_model_turn(mode):
|
||||
from artemis.agents.validator.action_execution import exec_action
|
||||
from artemis.mcp.action_executor import McpActionExecutor
|
||||
from artemis.mcp.action_types import ActionResult, ObserveResult
|
||||
|
||||
ctx = SimpleNamespace(
|
||||
data_engine=None, device=SimpleNamespace(device_width=720, device_height=1600)
|
||||
)
|
||||
receipt = {
|
||||
"sequence_results": [{"action": "click", "execution": "completed"}],
|
||||
"execution": "partial",
|
||||
}
|
||||
session = MagicMock()
|
||||
session.call = AsyncMock(
|
||||
return_value=ActionResult.failure("action_sequence", "Interrupted", metadata=receipt)
|
||||
)
|
||||
args = {"steps": steps()}
|
||||
if mode == "pro":
|
||||
action = {"action": "action_sequence", "args": args}
|
||||
ok, _ = await exec_action(ctx, session, action)
|
||||
assert not ok
|
||||
assert action["execution"] == "partial"
|
||||
from artemis.utils.task_tree import format_result_clean
|
||||
|
||||
text = format_result_clean({"status": "failed", "execution": [action]})
|
||||
else:
|
||||
executor = McpActionExecutor(ctx, actuator=MagicMock())
|
||||
executor._session_or_start = AsyncMock(return_value=session)
|
||||
executor._observe_after_action = AsyncMock(return_value=(ObserveResult(ok=True), True))
|
||||
result = await executor._execute_device_action(
|
||||
"action_sequence",
|
||||
"action_sequence",
|
||||
args,
|
||||
"gesture",
|
||||
SimpleNamespace(latest_screenshot=None, indexed_elements=[]),
|
||||
)
|
||||
assert result.status == "error"
|
||||
executor._observe_after_action.assert_awaited_once()
|
||||
text = result.text_summary
|
||||
assert '"execution": "partial"' in text
|
||||
assert '"execution": "completed"' in text
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pro_translation_and_wire_dispatch_preserve_exact_pixel_drag():
|
||||
from artemis.agents.operator.operator import OperatorNode
|
||||
from artemis.agents.validator.action_execution import exec_action
|
||||
|
||||
actuator = MockActuator(width=720, height=1600)
|
||||
node = object.__new__(OperatorNode)
|
||||
node.ctx = actuator.ctx
|
||||
request = {
|
||||
"start": [191, 1001],
|
||||
"end": [191, 1032],
|
||||
"coordinate_space": "pixel",
|
||||
"target_description": "timeline",
|
||||
}
|
||||
state = SimpleNamespace(operator_raw_data={"width": 720, "height": 1600})
|
||||
actions, error = node._translate_and_validate_tool_inner(
|
||||
{"name": "drag", "args": request}, state
|
||||
)
|
||||
assert error is None
|
||||
assert actions[0]["coordinates"] == [191, 1001, 191, 1032]
|
||||
assert actions[0]["normalized_coordinates"] == [265, 626, 265, 645]
|
||||
session = await ActionSession(build_action_server(actuator)).start()
|
||||
try:
|
||||
ok, _ = await exec_action(actuator.ctx, session, actions[0])
|
||||
assert ok
|
||||
assert actuator.action_history[-1]["end"] == (191, 1032)
|
||||
assert actions[0]["execution"] == "completed"
|
||||
finally:
|
||||
await session.aclose()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"change",
|
||||
[
|
||||
{"start": [True, 10]},
|
||||
{"duration_ms": 0},
|
||||
{"target_description": " "},
|
||||
{"end": None},
|
||||
{"coordinate_space": "screen"},
|
||||
],
|
||||
)
|
||||
def test_bad_drag_arguments_are_rejected(change):
|
||||
args = {"start": [10, 20], "end": [20, 30], "target_description": "slider", **change}
|
||||
with pytest.raises(ValueError):
|
||||
DragRequest.model_validate(args)
|
||||
|
||||
|
||||
def test_pixel_history_under_1000_still_uses_its_actual_screen_geometry():
|
||||
from artemis.utils.coordinates import normalize_step_actions
|
||||
|
||||
action = {
|
||||
"action": "drag",
|
||||
"coordinates": [191, 900, 191, 920],
|
||||
"coordinate_space": "pixel",
|
||||
"screen_size": [720, 1600],
|
||||
}
|
||||
step = normalize_step_actions({"action_taken": action})
|
||||
assert step["action_taken"]["coordinates"] == [191, 900, 191, 920]
|
||||
assert step["action_taken"]["normalized_coordinates"] == [265, 562, 265, 575]
|
||||
|
||||
|
||||
def test_overlay_uses_executed_pixels_instead_of_original_request_units():
|
||||
import io
|
||||
from PIL import Image
|
||||
from artemis.utils.visualization import overlay_action_on_screenshot
|
||||
|
||||
buffer = io.BytesIO()
|
||||
Image.new("RGB", (720, 1600), "white").save(buffer, format="PNG")
|
||||
action = {
|
||||
"action": "drag",
|
||||
"coordinates": [190, 900, 190, 930],
|
||||
"coordinate_space": "pixel",
|
||||
"args": {"start": [264, 563], "end": [264, 581], "coordinate_space": "normalized"},
|
||||
}
|
||||
drawn = overlay_action_on_screenshot(buffer.getvalue(), action)
|
||||
assert drawn is not None
|
||||
result = Image.open(io.BytesIO(drawn))
|
||||
red, green, blue = result.getpixel((190, 915))
|
||||
assert red > 150 and green < 120 and blue < 120
|
||||
assert min(result.getpixel((137, 1450))) > 240
|
||||
|
||||
|
||||
def test_sequence_overlay_does_not_draw_unexecuted_targets():
|
||||
from artemis.utils.visualization import draw_action_overlay_on_image
|
||||
|
||||
raw = b"unused original image"
|
||||
assert (
|
||||
draw_action_overlay_on_image(
|
||||
raw,
|
||||
"action_sequence",
|
||||
{
|
||||
"sequence_results": [
|
||||
{
|
||||
"action": "click",
|
||||
"coordinates": [190, 900],
|
||||
"coordinate_space": "pixel",
|
||||
"execution": "not_started",
|
||||
},
|
||||
]
|
||||
},
|
||||
)
|
||||
== raw
|
||||
)
|
||||
Reference in new issue
Block a user