"""Behavioral regressions for short-lived controls and measured scale movement.""" import asyncio from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock import pytest from artemis.mcp.action_server import build_action_server from artemis.mcp.action_session import ActionSession from artemis.mcp.actuators import MockActuator from artemis.mcp.gestures import DragRequest, plan_drag def test_pan_uses_decreasing_observed_time_scale_without_overshoot(): request = DragRequest( start=[190, 1059], coordinate_space="pixel", target_description="timeline", calibration={ "axis": "y", "mode": "pan", "anchors": [ {"position": 1075, "value": 38400}, {"position": 1187, "value": 37800}, ], "target_value": 37800, "indicator_position": 1215, }, ) planned = plan_drag(request, 720, 1600) assert planned["coordinates"] == [190, 1059, 190, 1087] assert planned["calibration"]["value_per_pixel"] == pytest.approx(-600 / 112) assert planned["calibration"]["verification"] == "pending_observation" @pytest.mark.parametrize( "axis,start,expected", [("x", [80, 200], [125, 200]), ("y", [200, 80], [200, 125])] ) def test_thumb_maps_value_to_the_requested_axis(axis, start, expected): planned = plan_drag( DragRequest( start=start, coordinate_space="pixel", target_description="slider", calibration={ "axis": axis, "mode": "thumb", "anchors": [ {"position": 100, "value": 0}, {"position": 200, "value": 100}, ], "target_value": 25, }, ), 720, 1600, ) assert planned["coordinates"] == [*start, *expected] @pytest.mark.asyncio async def test_pixel_drag_is_not_quantized_through_normalized_coordinates(): actuator = MockActuator(width=720, height=1600) session = await ActionSession(build_action_server(actuator)).start() try: result = await session.call( "drag", { "start": [191, 1001], "end": [191, 1032], "coordinate_space": "pixel", "target_description": "timeline", }, ) assert result.ok assert result.metadata["coordinates"] == [191, 1001, 191, 1032] assert actuator.action_history == [ { "action": "swipe", "start": (191, 1001), "end": (191, 1032), "duration_ms": 300, } ] finally: await session.aclose() def steps(): return [ {"action": "click", "target": [200, 900], "target_description": "open scale"}, {"action": "wait", "time_in_ms": 1}, { "action": "drag", "start": [190, 1001], "end": [190, 1032], "coordinate_space": "pixel", "target_description": "scale handle", }, ] @pytest.mark.asyncio async def test_sequence_has_no_observation_between_inputs_and_returns_member_receipts(): actuator = MockActuator(width=720, height=1600) actuator.controller.tap_at = AsyncMock(return_value=MagicMock(error=None)) actuator.controller.swipe_coords = AsyncMock(return_value=None) actuator.take_screenshot = AsyncMock() result = await actuator.action_sequence(steps()) assert result.ok actuator.controller.tap_at.assert_awaited_once_with(144, 1440) actuator.controller.swipe_coords.assert_awaited_once_with(190, 1001, 190, 1032, 300) actuator.take_screenshot.assert_not_awaited() receipts = result.metadata["sequence_results"] assert [r["execution"] for r in receipts] == ["completed"] * 3 assert receipts[0]["ended_ms"] <= receipts[1]["started_ms"] <= receipts[2]["started_ms"] @pytest.mark.asyncio async def test_late_invalid_member_prevents_even_the_first_input(): actuator = MockActuator(width=720, height=1600) actuator.controller.tap_at = AsyncMock() invalid = steps() invalid[-1]["end"] = [720, 1032] result = await actuator.action_sequence(invalid) assert not result.ok and result.code.value == "INVALID_ARGS" actuator.controller.tap_at.assert_not_awaited() @pytest.mark.asyncio async def test_failure_keeps_completed_members_and_does_not_dispatch_the_remainder(): actuator = MockActuator(width=720, height=1600) actuator.controller.tap_at = AsyncMock( side_effect=[MagicMock(error=None), RuntimeError("transport lost")] ) member = {"action": "click", "target": [200, 900], "target_description": "control"} result = await actuator.action_sequence([member, member, member]) assert not result.ok assert actuator.controller.tap_at.await_count == 2 assert [r["execution"] for r in result.metadata["sequence_results"]] == [ "completed", "unknown", "not_started", ] assert result.metadata["execution"] == "partial" @pytest.mark.asyncio async def test_cancellation_is_not_swallowed_or_retried(): actuator = MockActuator(width=720, height=1600) actuator.ctx.data_engine = MagicMock(current_step_id="step") actuator.controller.swipe_coords = AsyncMock(side_effect=asyncio.CancelledError) with pytest.raises(asyncio.CancelledError): await actuator.action_sequence(steps()) assert len(actuator.action_history) == 1 assert actuator.controller.swipe_coords.await_count == 1 receipts = [ c.kwargs["payload"]["receipt"] for c in actuator.ctx.data_engine.record_trace.call_args_list if "receipt" in c.kwargs["payload"] ] assert [r["execution"] for r in receipts] == ["completed", "completed", "unknown"] @pytest.mark.asyncio async def test_pause_at_member_boundary_does_not_dispatch_later_inputs(tmp_path): from artemis.runtime.task_control import TaskControl actuator = MockActuator(width=720, height=1600) control = object.__new__(TaskControl) control.directory = tmp_path actuator.ctx.task_control = control async def tap(*_): (tmp_path / "pause.request").write_text("{}") return SimpleNamespace(error=None) actuator.controller.tap_at = AsyncMock(side_effect=tap) actuator.controller.swipe_coords = AsyncMock() result = await actuator.action_sequence(steps()) assert not result.ok assert result.metadata["completed_steps"] == 1 assert result.metadata["sequence_results"][1]["execution"] == "not_started" actuator.controller.swipe_coords.assert_not_awaited() @pytest.mark.asyncio async def test_over_limit_is_rejected_before_dispatch(monkeypatch): monkeypatch.setattr( "artemis.config.agent.load_agent_config", lambda: SimpleNamespace( pro=SimpleNamespace(execution=SimpleNamespace(max_burst_actions=2)), ), ) actuator = MockActuator(width=720, height=1600) result = await actuator.action_sequence(steps()) assert not result.ok and "limit of 2" in result.message assert actuator.action_history == [] @pytest.mark.parametrize("values", [(0, 0), (1e308, -1e308), (float("nan"), 1), (True, 5)]) def test_degenerate_and_nonfinite_calibration_is_rejected(values): with pytest.raises(ValueError): request = DragRequest( start=[100, 100], coordinate_space="pixel", target_description="timeline", calibration={ "axis": "y", "mode": "pan", "indicator_position": 100, "anchors": [ {"position": 100, "value": values[0]}, {"position": 200, "value": values[1]}, ], "target_value": 3, }, ) plan_drag(request, 720, 1600) def test_flash_and_pro_schemas_preserve_null_for_calculated_endpoints(): from langchain_core.utils.function_calling import convert_to_openai_tool from artemis.mcp.action_specs import tool_declaration, operator_shell_tool flash = tool_declaration("drag").parameters pro = convert_to_openai_tool(operator_shell_tool("drag"))["function"]["parameters"] assert flash == pro assert {"type": "null"} in flash["properties"]["end"]["anyOf"] @pytest.mark.asyncio async def test_conflicting_endpoint_has_actionable_error_and_no_input(): actuator = MockActuator(width=720, height=1600) result = await actuator.drag( start=[190, 1200], end=[190, 1100], coordinate_space="pixel", target_description="timeline", calibration={ "axis": "y", "mode": "pan", "anchors": [{"position": 1000, "value": 60}, {"position": 1100, "value": 0}], "target_value": 30, "indicator_position": 1050, }, ) assert not result.ok and "set end=null" in result.message assert actuator.action_history == [] @pytest.mark.asyncio @pytest.mark.parametrize("mode", ["flash", "pro"]) async def test_each_mode_exposes_the_receipt_to_its_next_model_turn(mode): from artemis.agents.validator.action_execution import exec_action from artemis.mcp.action_executor import McpActionExecutor from artemis.mcp.action_types import ActionResult, ObserveResult ctx = SimpleNamespace( data_engine=None, device=SimpleNamespace(device_width=720, device_height=1600) ) receipt = { "sequence_results": [{"action": "click", "execution": "completed"}], "execution": "partial", } session = MagicMock() session.call = AsyncMock( return_value=ActionResult.failure("action_sequence", "Interrupted", metadata=receipt) ) args = {"steps": steps()} if mode == "pro": action = {"action": "action_sequence", "args": args} ok, _ = await exec_action(ctx, session, action) assert not ok assert action["execution"] == "partial" from artemis.utils.task_tree import format_result_clean text = format_result_clean({"status": "failed", "execution": [action]}) else: executor = McpActionExecutor(ctx, actuator=MagicMock()) executor._session_or_start = AsyncMock(return_value=session) executor._observe_after_action = AsyncMock(return_value=(ObserveResult(ok=True), True)) result = await executor._execute_device_action( "action_sequence", "action_sequence", args, "gesture", SimpleNamespace(latest_screenshot=None, indexed_elements=[]), ) assert result.status == "error" executor._observe_after_action.assert_awaited_once() text = result.text_summary assert '"execution": "partial"' in text assert '"execution": "completed"' in text @pytest.mark.asyncio async def test_pro_translation_and_wire_dispatch_preserve_exact_pixel_drag(): from artemis.agents.operator.operator import OperatorNode from artemis.agents.validator.action_execution import exec_action actuator = MockActuator(width=720, height=1600) node = object.__new__(OperatorNode) node.ctx = actuator.ctx request = { "start": [191, 1001], "end": [191, 1032], "coordinate_space": "pixel", "target_description": "timeline", } state = SimpleNamespace(operator_raw_data={"width": 720, "height": 1600}) actions, error = node._translate_and_validate_tool_inner( {"name": "drag", "args": request}, state ) assert error is None assert actions[0]["coordinates"] == [191, 1001, 191, 1032] assert actions[0]["normalized_coordinates"] == [265, 626, 265, 645] session = await ActionSession(build_action_server(actuator)).start() try: ok, _ = await exec_action(actuator.ctx, session, actions[0]) assert ok assert actuator.action_history[-1]["end"] == (191, 1032) assert actions[0]["execution"] == "completed" finally: await session.aclose() @pytest.mark.parametrize( "change", [ {"start": [True, 10]}, {"duration_ms": 0}, {"target_description": " "}, {"end": None}, {"coordinate_space": "screen"}, ], ) def test_bad_drag_arguments_are_rejected(change): args = {"start": [10, 20], "end": [20, 30], "target_description": "slider", **change} with pytest.raises(ValueError): DragRequest.model_validate(args) def test_pixel_history_under_1000_still_uses_its_actual_screen_geometry(): from artemis.utils.coordinates import normalize_step_actions action = { "action": "drag", "coordinates": [191, 900, 191, 920], "coordinate_space": "pixel", "screen_size": [720, 1600], } step = normalize_step_actions({"action_taken": action}) assert step["action_taken"]["coordinates"] == [191, 900, 191, 920] assert step["action_taken"]["normalized_coordinates"] == [265, 562, 265, 575] def test_overlay_uses_executed_pixels_instead_of_original_request_units(): import io from PIL import Image from artemis.utils.visualization import overlay_action_on_screenshot buffer = io.BytesIO() Image.new("RGB", (720, 1600), "white").save(buffer, format="PNG") action = { "action": "drag", "coordinates": [190, 900, 190, 930], "coordinate_space": "pixel", "args": {"start": [264, 563], "end": [264, 581], "coordinate_space": "normalized"}, } drawn = overlay_action_on_screenshot(buffer.getvalue(), action) assert drawn is not None result = Image.open(io.BytesIO(drawn)) red, green, blue = result.getpixel((190, 915)) assert red > 150 and green < 120 and blue < 120 assert min(result.getpixel((137, 1450))) > 240 def test_sequence_overlay_does_not_draw_unexecuted_targets(): from artemis.utils.visualization import draw_action_overlay_on_image raw = b"unused original image" assert ( draw_action_overlay_on_image( raw, "action_sequence", { "sequence_results": [ { "action": "click", "coordinates": [190, 900], "coordinate_space": "pixel", "execution": "not_started", }, ] }, ) == raw )