chenmingxuan/Artemis-Eapil
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 32 Wiki Activity

fix: guard transient controls and verify timeline endpoints

Browse Source
This commit is contained in:
chenmingxuan committed 2026-11-02 13:54:50 +08:00
1 parent 479a185192
commit 00e62a663a
29 files changed
+1150 -90
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/fixtures/action_surfaces/action_server_manifest.json
Vendored
+83 -7
View File
@@ -12,13 +12,22 @@
"type": "string"
},
"target": {
"items": {
"type": "integer"
},
"maxItems": 2,
"minItems": 2,
"title": "Target",
"type": "array"
"anyOf": [
{
"items": {
"type": "integer"
},
"maxItems": 2,
"minItems": 2,
"type": "array"
},
{
"minLength": 1,
"type": "string"
}
],
"description": "Prefer an exact observed resource_id: resolve its live bounds before tapping, refusing absent/ambiguous targets. Otherwise use [x,y] in coordinate_space; never mix screenshot pixels with normalized coordinates.",
"title": "Target"
},
"target_description": {
"minLength": 1,
@@ -33,6 +42,35 @@
],
"title": "Coordinate Space",
"type": "string"
},
"times": {
"default": 1,
"description": "Consecutive taps within this ONE member, e.g. 10 for a scale +/- button; no model turns between taps.",
"maximum": 10,
"minimum": 1,
"title": "Times",
"type": "integer"
},
"delay_ms": {
"default": 100,
"maximum": 1000,
"minimum": 0,
"title": "Delay Ms",
"type": "integer"
},
"unless_visible": {
"anyOf": [
{
"minLength": 1,
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Exact previously observed resource_id inside a toggled panel (not the opener). Skip this opener click when that control is already visible; otherwise click. Avoid closing a still-open panel.",
"title": "Unless Visible"
}
},
"required": [
@@ -162,6 +200,20 @@
"default": null,
"description": "Computes the endpoint from observed anchors; then set end=null."
},
"require_visible": {
"anyOf": [
{
"minLength": 1,
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Exact previously observed resource_id of a panel control. Check live visibility before dragging; refuse input if absent. Use for auto-hiding controls.",
"title": "Require Visible"
},
"action": {
"const": "drag",
"title": "Action",
@@ -467,6 +519,18 @@
}
],
"default": null
},
"require_visible": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"title": "Require Visible"
}
},
"required": [
@@ -585,6 +649,18 @@
"default": 100,
"title": "Delay Ms",
"type": "integer"
},
"expected_resource_id": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"title": "Expected Resource Id"
}
},
"required": [
tests/fixtures/action_surfaces/operator_shells.json
Vendored
+76 -10
View File
@@ -3,7 +3,7 @@
"type": "function",
"function": {
"name": "click",
"description": "[ACTION] Click on the target location on the screen (supports element index or absolute normalized coordinates).",
"description": "[ACTION] Click on the target location on the screen (supports element index or absolute normalized coordinates). Indexed resource targets are checked against live bounds before input; missing/moved targets refuse the tap. For floating or auto-hiding panels, use a guarded action_sequence with opener and target in the same call; an open panel in a screenshot may already have expired.",
"parameters": {
"properties": {
"target": {
@@ -54,7 +54,7 @@
"type": "function",
"function": {
"name": "action_sequence",
"description": "[ACTION] Execute a short sequence of click, wait and drag steps in ONE call, in order, without model turns or screenshots between members. Use for temporary overlays: open the control, wait briefly, then operate it before it disappears. Use observed coordinates, or positions learned from an earlier observation of the same restored layout; element indices are not accepted. Each click/drag needs target_description. All members are validated before any input. The configured fast-action limit applies. The first failure stops the remainder; receipts identify completed and unexecuted members. Never replay the whole sequence after a partial result. Verify the final screen before claiming the intended result.",
"description": "[ACTION] Execute a short sequence of click, wait and drag steps in ONE call, in order, without model turns between members; explicit visibility guards may read the live UI locally. Use for temporary overlays: open the control, wait briefly, then operate it before it disappears. For clicks prefer target=the exact observed resource_id string: live bounds are resolved locally, refusing absent or ambiguous targets. Otherwise use observed [x,y] in the declared coordinate_space; do not confuse original screenshot pixels with 0..1000 normalized coordinates. Element indices are not accepted. Each click/drag needs target_description. All members are validated before any input. The configured fast-action limit applies. The first failure stops the remainder; receipts identify completed and unexecuted members. Never replay the whole sequence after a partial result. Every wait counts as a member. click.times (1..10) performs consecutive taps within ONE member, like the ordinary click tool; prefer this over expanding repeated +/- taps into many members/model turns. Each resource-targeted tap is revalidated locally; missing targets stop remaining taps and receipts preserve completed_taps. For maximum/minimum prefer observed native +/- controls, confirming that further same-direction adjustment stops changing handle/ticks. An estimated drag endpoint can be imprecise. For every auto-hiding adjustment, including the reverse direction, keep opener/wait/operation in one call. A prior screenshot does not prove the panel is still open. Use click.unless_visible and drag.require_visible with an observed panel resource_id to avoid toggling an open panel closed or dragging its underlying content. Never invent resource IDs. Verify actual ticks/handle endpoints before claiming maximum or minimum.",
"parameters": {
"properties": {
"steps": {
@@ -70,13 +70,22 @@
"type": "string"
},
"target": {
"items": {
"type": "integer"
},
"maxItems": 2,
"minItems": 2,
"title": "Target",
"type": "array"
"anyOf": [
{
"items": {
"type": "integer"
},
"maxItems": 2,
"minItems": 2,
"type": "array"
},
{
"minLength": 1,
"type": "string"
}
],
"description": "Prefer an exact observed resource_id: resolve its live bounds before tapping, refusing absent/ambiguous targets. Otherwise use [x,y] in coordinate_space; never mix screenshot pixels with normalized coordinates.",
"title": "Target"
},
"target_description": {
"minLength": 1,
@@ -91,6 +100,35 @@
],
"title": "Coordinate Space",
"type": "string"
},
"times": {
"default": 1,
"description": "Consecutive taps within this ONE member, e.g. 10 for a scale +/- button; no model turns between taps.",
"maximum": 10,
"minimum": 1,
"title": "Times",
"type": "integer"
},
"delay_ms": {
"default": 100,
"maximum": 1000,
"minimum": 0,
"title": "Delay Ms",
"type": "integer"
},
"unless_visible": {
"anyOf": [
{
"minLength": 1,
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Exact previously observed resource_id inside a toggled panel (not the opener). Skip this opener click when that control is already visible; otherwise click. Avoid closing a still-open panel.",
"title": "Unless Visible"
}
},
"required": [
@@ -235,6 +273,20 @@
"default": null,
"description": "Computes the endpoint from observed anchors; then set end=null."
},
"require_visible": {
"anyOf": [
{
"minLength": 1,
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Exact previously observed resource_id of a panel control. Check live visibility before dragging; refuse input if absent. Use for auto-hiding controls.",
"title": "Require Visible"
},
"action": {
"const": "drag",
"title": "Action",
@@ -273,6 +325,8 @@
}
]
},
"maxItems": 10,
"minItems": 1,
"type": "array"
}
},
@@ -287,7 +341,7 @@
"type": "function",
"function": {
"name": "drag",
"description": "[ACTION] Precisely drag a local slider or timeline. Use explicit start/end or calibration from TWO observed numeric scale anchors. coordinate_space is normalized (0..1000) or pixel (the current screenshot's actual pixels); pixel avoids quantization on a zoomed scale. Default duration_ms=300: slower is not always more precise and may trigger long-press recognition. Do not overshoot the endpoint automatically. For calibration, axis=x/y; mode=thumb moves the handle to target_value, mode=pan shifts the scale under indicator_position. Use one numeric unit, e.g. seconds since midnight, and two anchors from the CURRENT zoom/layout. Provide exactly one of end or calibration. Returned geometry is a plan/dispatch receipt, not proof of the selected value: compare the post-action scale/value and correct only the observed residual; zoom further if the pixel resolution is insufficient.",
"description": "[ACTION] Precisely drag a local slider or timeline. Use explicit start/end or calibration from TWO observed numeric scale anchors. coordinate_space is normalized (0..1000) or pixel (the current screenshot's actual pixels); pixel avoids quantization on a zoomed scale. Default duration_ms=300. If a panned timeline coasts past the release position, try a slower pan (e.g. 1500ms), then wait for its ticks to stop before measuring the residual. A transient release-time bubble can disagree with the final fixed-pointer scale; do not treat it as final position proof. A permanently visible timeline can be panned independently of its zoom popup. Do not overshoot the endpoint automatically. For calibration, axis=x/y; mode=thumb moves the handle to target_value, mode=pan shifts the scale under indicator_position. Use one numeric unit, e.g. seconds since midnight, and two anchors from the CURRENT zoom/layout. Provide exactly one of end or calibration. Returned geometry is a plan/dispatch receipt, not proof of the selected value: compare the post-action scale/value and correct only the observed residual; zoom further if the pixel resolution is insufficient.",
"parameters": {
"properties": {
"start": {
@@ -410,6 +464,18 @@
],
"default": null,
"description": "Observed numeric anchors that compute the endpoint. When supplied, end MUST be null; otherwise set calibration null."
},
"require_visible": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Previously observed panel resource_id; refuse the drag if it is no longer visible."
}
},
"required": [
tests/fixtures/action_surfaces/tool_declarations.json
Vendored
+76 -10
View File
@@ -3,7 +3,7 @@
"type": "function",
"function": {
"name": "click",
"description": "[ACTION] Click on the target location on the screen (supports element index or absolute normalized coordinates).",
"description": "[ACTION] Click on the target location on the screen (supports element index or absolute normalized coordinates). Indexed resource targets are checked against live bounds before input; missing/moved targets refuse the tap. For floating or auto-hiding panels, use a guarded action_sequence with opener and target in the same call; an open panel in a screenshot may already have expired.",
"parameters": {
"type": "object",
"properties": {
@@ -315,7 +315,7 @@
"type": "function",
"function": {
"name": "drag",
"description": "[ACTION] Precisely drag a local slider or timeline. Use explicit start/end or calibration from TWO observed numeric scale anchors. coordinate_space is normalized (0..1000) or pixel (the current screenshot's actual pixels); pixel avoids quantization on a zoomed scale. Default duration_ms=300: slower is not always more precise and may trigger long-press recognition. Do not overshoot the endpoint automatically. For calibration, axis=x/y; mode=thumb moves the handle to target_value, mode=pan shifts the scale under indicator_position. Use one numeric unit, e.g. seconds since midnight, and two anchors from the CURRENT zoom/layout. Provide exactly one of end or calibration. Returned geometry is a plan/dispatch receipt, not proof of the selected value: compare the post-action scale/value and correct only the observed residual; zoom further if the pixel resolution is insufficient.",
"description": "[ACTION] Precisely drag a local slider or timeline. Use explicit start/end or calibration from TWO observed numeric scale anchors. coordinate_space is normalized (0..1000) or pixel (the current screenshot's actual pixels); pixel avoids quantization on a zoomed scale. Default duration_ms=300. If a panned timeline coasts past the release position, try a slower pan (e.g. 1500ms), then wait for its ticks to stop before measuring the residual. A transient release-time bubble can disagree with the final fixed-pointer scale; do not treat it as final position proof. A permanently visible timeline can be panned independently of its zoom popup. Do not overshoot the endpoint automatically. For calibration, axis=x/y; mode=thumb moves the handle to target_value, mode=pan shifts the scale under indicator_position. Use one numeric unit, e.g. seconds since midnight, and two anchors from the CURRENT zoom/layout. Provide exactly one of end or calibration. Returned geometry is a plan/dispatch receipt, not proof of the selected value: compare the post-action scale/value and correct only the observed residual; zoom further if the pixel resolution is insufficient.",
"parameters": {
"properties": {
"start": {
@@ -438,6 +438,18 @@
],
"default": null,
"description": "Observed numeric anchors that compute the endpoint. When supplied, end MUST be null; otherwise set calibration null."
},
"require_visible": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Previously observed panel resource_id; refuse the drag if it is no longer visible."
}
},
"required": [
@@ -452,7 +464,7 @@
"type": "function",
"function": {
"name": "action_sequence",
"description": "[ACTION] Execute a short sequence of click, wait and drag steps in ONE call, in order, without model turns or screenshots between members. Use for temporary overlays: open the control, wait briefly, then operate it before it disappears. Use observed coordinates, or positions learned from an earlier observation of the same restored layout; element indices are not accepted. Each click/drag needs target_description. All members are validated before any input. The configured fast-action limit applies. The first failure stops the remainder; receipts identify completed and unexecuted members. Never replay the whole sequence after a partial result. Verify the final screen before claiming the intended result.",
"description": "[ACTION] Execute a short sequence of click, wait and drag steps in ONE call, in order, without model turns between members; explicit visibility guards may read the live UI locally. Use for temporary overlays: open the control, wait briefly, then operate it before it disappears. For clicks prefer target=the exact observed resource_id string: live bounds are resolved locally, refusing absent or ambiguous targets. Otherwise use observed [x,y] in the declared coordinate_space; do not confuse original screenshot pixels with 0..1000 normalized coordinates. Element indices are not accepted. Each click/drag needs target_description. All members are validated before any input. The configured fast-action limit applies. The first failure stops the remainder; receipts identify completed and unexecuted members. Never replay the whole sequence after a partial result. Every wait counts as a member. click.times (1..10) performs consecutive taps within ONE member, like the ordinary click tool; prefer this over expanding repeated +/- taps into many members/model turns. Each resource-targeted tap is revalidated locally; missing targets stop remaining taps and receipts preserve completed_taps. For maximum/minimum prefer observed native +/- controls, confirming that further same-direction adjustment stops changing handle/ticks. An estimated drag endpoint can be imprecise. For every auto-hiding adjustment, including the reverse direction, keep opener/wait/operation in one call. A prior screenshot does not prove the panel is still open. Use click.unless_visible and drag.require_visible with an observed panel resource_id to avoid toggling an open panel closed or dragging its underlying content. Never invent resource IDs. Verify actual ticks/handle endpoints before claiming maximum or minimum.",
"parameters": {
"properties": {
"steps": {
@@ -468,13 +480,22 @@
"type": "string"
},
"target": {
"items": {
"type": "integer"
},
"maxItems": 2,
"minItems": 2,
"title": "Target",
"type": "array"
"anyOf": [
{
"items": {
"type": "integer"
},
"maxItems": 2,
"minItems": 2,
"type": "array"
},
{
"minLength": 1,
"type": "string"
}
],
"description": "Prefer an exact observed resource_id: resolve its live bounds before tapping, refusing absent/ambiguous targets. Otherwise use [x,y] in coordinate_space; never mix screenshot pixels with normalized coordinates.",
"title": "Target"
},
"target_description": {
"minLength": 1,
@@ -489,6 +510,35 @@
],
"title": "Coordinate Space",
"type": "string"
},
"times": {
"default": 1,
"description": "Consecutive taps within this ONE member, e.g. 10 for a scale +/- button; no model turns between taps.",
"maximum": 10,
"minimum": 1,
"title": "Times",
"type": "integer"
},
"delay_ms": {
"default": 100,
"maximum": 1000,
"minimum": 0,
"title": "Delay Ms",
"type": "integer"
},
"unless_visible": {
"anyOf": [
{
"minLength": 1,
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Exact previously observed resource_id inside a toggled panel (not the opener). Skip this opener click when that control is already visible; otherwise click. Avoid closing a still-open panel.",
"title": "Unless Visible"
}
},
"required": [
@@ -633,6 +683,20 @@
"default": null,
"description": "Computes the endpoint from observed anchors; then set end=null."
},
"require_visible": {
"anyOf": [
{
"minLength": 1,
"type": "string"
},
{
"type": "null"
}
],
"default": null,
"description": "Exact previously observed resource_id of a panel control. Check live visibility before dragging; refuse input if absent. Use for auto-hiding controls.",
"title": "Require Visible"
},
"action": {
"const": "drag",
"title": "Action",
@@ -671,6 +735,8 @@
}
]
},
"maxItems": 10,
"minItems": 1,
"type": "array"
}
},
tests/unit/agents/test_flash_runner.py
+3 -1
View File
@@ -134,7 +134,9 @@ async def test_fallback_observation_renders_the_indexed_ui_list(mock_context, tm
assert post == _POST_IMG
assert isinstance(exec_result.ui_elements_text, str)
assert exec_result.ui_elements_text == "[1] Text: 'Settings' | Bounds: [37,50][962,112]"
assert exec_result.ui_elements_text == (
"[1] Text: 'Settings' | Bounds: [37,50][962,112] | resource_id: 'android:id/title'"
)
assert "{'text'" not in exec_result.ui_elements_text
# State refreshed exactly as the executor's success path does.
assert state.indexed_elements[0]["index"] == 1
tests/unit/agents/test_validator.py
+43
View File
@@ -23,6 +23,7 @@ feeds the safety-net XML validation.
import json
from unittest.mock import AsyncMock, MagicMock, Mock, patch
from uuid import uuid4
from langchain_core.messages import AIMessage
import pytest
@@ -309,6 +310,48 @@ async def test_validator_burst_success_executes_every_member(
assert len(mock_mcp.calls_for("click")) == 2
@pytest.mark.asyncio
@pytest.mark.parametrize("post_image", ["screenshot", "new-sequence-image"])
async def test_sequence_captures_once_after_whole_burst_and_retains_equal_hash(
mock_mcp, mock_context, temp_screenshot, post_image
):
events = []
def action(name, args):
events.append(name)
return True, ""
def screenshot():
events.append("capture")
return "ZHVtbXlfZGF0YQ=="
mock_mcp.action_handler = action
mock_mcp.screenshot = screenshot
mock_context.data_engine.get_or_create_image.return_value = post_image
mock_context.data_engine.get_image_path.return_value = temp_screenshot
state = DummyState(
structured_decisions=json.dumps(
[
{
"action": "action_sequence",
"args": {"steps": [{"action": "wait", "time_in_ms": 1}]},
},
{"action": "tap", "coordinates": [105, 205]},
]
),
latest_screenshot=temp_screenshot,
current_step_id=str(uuid4()),
)
result = await ValidatorNode(mock_context)(state)
assert result["last_execution_result"]["status"] == "dispatched"
assert events == ["action_sequence", "click", "capture"]
assert mock_context.data_engine.update_step_execution_result.call_args.kwargs == {
"post_image_name": post_image
}
@pytest.mark.asyncio
async def test_validator_success_closes_open_incident(mock_mcp, mock_context, temp_screenshot):
"""A successful turn closes whatever incident was still open."""
tests/unit/mcp/test_action_executor_semantics.py
+6 -1
View File
@@ -111,7 +111,12 @@ def test_click_index_resolves_to_the_elements_center_and_records_observed_fields
)
assert wire_name == "click"
# 540/1080 -> 500, 1248/2400 -> 520 (normalized 0-1000)
assert wire_args == {"target": [500, 520], "times": 1, "delay_ms": 100}
assert wire_args == {
"target": [500, 520],
"times": 1,
"delay_ms": 100,
"expected_resource_id": "android:id/title",
}
assert recorded == {
"target_text": "Wi-Fi",
"target_bounds": [37, 480, 963, 560],
tests/unit/mcp/test_action_manifest.py
+12 -1
View File
@@ -57,7 +57,18 @@ SCOPED_PROMPT_FILES = [
# decision.
#: Backticked identifiers in prompts that are parameters or note keys, not tools.
NON_TOOL_IDENTIFIERS = frozenset(
{"analysis", "task_plan", "tool_enum", "target_description", "report_note_key"}
{
"analysis",
"task_plan",
"tool_enum",
"target_description",
"report_note_key",
"unless_visible",
"require_visible",
"sequence_results",
"skipped",
"not_started",
}
)
_TOOL_REF = re.compile(r"(?:`([a-z][a-z0-9_]{2,})`|\b([a-z][a-z0-9_]{2,})\()")
tests/unit/mcp/test_action_observation_settling.py
+14
View File
@@ -90,6 +90,20 @@ async def test_old_bitmap_with_new_hierarchy_is_never_marked_settled(tmp_path):
assert session.observe.await_count == 4
@pytest.mark.asyncio
async def test_sequence_preserves_immediate_popup_evidence_without_claiming_stability(tmp_path):
popup = _observation(_screen(tmp_path, "popup.jpg"), "缩放面板")
session = Mock()
session.observe = AsyncMock(return_value=popup)
executor = McpActionExecutor(Mock(), actuator=Mock())
observed, settled = await executor._observe_after_action(session, "action_sequence", None)
assert observed is popup
assert settled is None
session.observe.assert_awaited_once_with(settle_ms=0)
@pytest.mark.asyncio
async def test_read_only_action_keeps_single_observation(tmp_path):
home = _screen(tmp_path, "home.jpg")
tests/unit/mcp/test_gestures.py
+330 -1
View File
@@ -101,6 +101,335 @@ def steps():
]
def test_both_observation_lists_expose_real_resource_ids_without_changing_indices():
from artemis.utils.visualization import (
format_minimal_list_with_elements,
format_minimal_list_with_points,
)
nodes = [
{"text": "plus", "resource-id": "test.app:id/plus", "bounds": "[10,20][30,40]"},
{"text": "minus", "resource_id": "test.app:id/minus", "bounds": "[10,60][30,80]"},
]
for formatter in (format_minimal_list_with_points, format_minimal_list_with_elements):
text, _, labels = formatter(nodes, 100, 100)
assert (
"[1] Text: 'plus' | Bounds: [100,200][300,400] | resource_id: 'test.app:id/plus'"
in text
)
assert "[2] Text: 'minus'" in text and "resource_id: 'test.app:id/minus'" in text
assert labels == ["1", "2"]
@pytest.mark.asyncio
async def test_live_resource_click_uses_current_bounds_and_reuses_guard_observation():
actuator = MockActuator(width=720, height=1600)
panel, opener = "test.app:id/panel", "test.app:id/opener"
actuator.controller.get_ui_elements = AsyncMock(
return_value=[
{"resource_id": opener, "bounds": [100, 1400, 200, 1500]},
]
)
actuator.controller.tap_at = AsyncMock(return_value=SimpleNamespace(error=None))
result = await actuator.action_sequence(
[
{
"action": "click",
"target": opener,
"target_description": "open",
"unless_visible": panel,
}
]
)
assert result.ok
actuator.controller.tap_at.assert_awaited_once_with(150, 1450)
actuator.controller.get_ui_elements.assert_awaited_once()
receipt = result.metadata["sequence_results"][0]
assert receipt["coordinates"] == [150, 1450]
assert receipt["target_resource_id"] == opener
@pytest.mark.asyncio
@pytest.mark.parametrize("matches", [0, 2])
async def test_missing_or_ambiguous_resource_click_never_guesses_coordinates(matches):
actuator = MockActuator(width=720, height=1600)
target = "test.app:id/control"
actuator.controller.get_ui_elements = AsyncMock(
return_value=[
{
"resource_id": target if i < matches else "test.app:id/other",
"bounds": [10, 20, 30, 40],
}
for i in range(2)
]
)
actuator.controller.tap_at = AsyncMock()
result = await actuator.action_sequence(
[{"action": "click", "target": target, "target_description": "control"}]
)
assert not result.ok and result.code.value == "TARGET_NOT_FOUND"
assert result.metadata["execution"] == "not_started"
actuator.controller.tap_at.assert_not_awaited()
@pytest.mark.asyncio
@pytest.mark.parametrize(
"resource_id,bounds,success",
[
("test.app:id/control", [100, 1400, 200, 1500], True),
("test.app:id/other", [100, 1400, 200, 1500], False),
("test.app:id/control", [300, 1400, 400, 1500], False),
],
)
async def test_indexed_click_checks_live_identity_at_original_position(
resource_id, bounds, success
):
actuator = MockActuator(width=720, height=1600)
actuator.controller.get_ui_elements = AsyncMock(
return_value=[{"resource_id": resource_id, "bounds": bounds}]
)
actuator.controller.tap_at = AsyncMock(return_value=SimpleNamespace(error=None))
result = await actuator.click(217, 906, expected_resource_id="test.app:id/control")
assert result.ok is success
assert actuator.controller.tap_at.await_count == int(success)
if success:
from artemis.agents.flash.runner import FlashRunner
assert result.metadata["pixel_coordinates"] == [156, 1449]
assert "coordinates" not in result.metadata
runner = object.__new__(FlashRunner)
record = runner._build_action_record(
"click", {"target": [217, 906]}, SimpleNamespace(metadata=result.metadata)
)
assert record["coordinates"] == [217, 906]
assert record["coordinate_space"] == "normalized"
assert record["completed_taps"] == 1
assert record["pixel_coordinates"] == [156, 1449]
else:
assert result.metadata["execution"] == "not_started"
assert result.metadata["side_effect"] == "none"
@pytest.mark.asyncio
async def test_coordinate_only_click_keeps_existing_direct_dispatch():
actuator = MockActuator(width=720, height=1600)
actuator.controller.get_ui_elements = AsyncMock()
result = await actuator.click(217, 906)
assert result.ok
actuator.controller.get_ui_elements.assert_not_awaited()
def test_flash_and_pro_forward_observed_resource_identity_without_inventing_it():
from artemis.mcp.action_executor import McpActionExecutor
from artemis.mcp.action_names import to_canonical_call
executor = object.__new__(McpActionExecutor)
executor._resolve_target = MagicMock(
return_value=([217, 906], {"target_resource_id": "test.app:id/control"})
)
name, args, _, _ = executor._translate("click", {"target": 1}, MagicMock())
assert name == "click" and args["expected_resource_id"] == "test.app:id/control"
_, args = to_canonical_call(
{
"action": "tap",
"normalized_coordinates": [217, 906],
"resource_id": "test.app:id/control",
}
)
assert args["expected_resource_id"] == "test.app:id/control"
@pytest.mark.asyncio
async def test_repeated_resource_taps_are_one_bounded_sequence_member():
actuator = MockActuator(width=720, height=1600)
actuator.controller.get_ui_elements = AsyncMock(
return_value=[{"resource_id": "test.app:id/plus", "bounds": [10, 20, 30, 40]}]
)
actuator.controller.tap_at = AsyncMock(return_value=SimpleNamespace(error=None))
result = await actuator.action_sequence(
[
{
"action": "click",
"target": "test.app:id/plus",
"target_description": "plus",
"times": 10,
"delay_ms": 50,
}
]
)
assert result.ok and result.metadata["completed_steps"] == 1
assert actuator.controller.tap_at.await_count == 10
actuator.controller.tap_at.assert_awaited_with(20, 30)
assert actuator.controller.get_ui_elements.await_count == 10
receipt = result.metadata["sequence_results"][0]
assert receipt["times"] == receipt["completed_taps"] == 10
@pytest.mark.asyncio
@pytest.mark.parametrize("sequence", [True, False])
async def test_repeated_native_taps_stop_when_target_disappears(sequence):
actuator = MockActuator(width=720, height=1600)
actuator.controller.get_ui_elements = AsyncMock(
side_effect=[
[{"resource_id": "test.app:id/plus", "bounds": [10, 20, 30, 40]}],
[{"resource_id": "test.app:id/background", "bounds": [0, 0, 720, 1600]}],
]
)
actuator.controller.tap_at = AsyncMock(return_value=SimpleNamespace(error=None))
if sequence:
result = await actuator.action_sequence(
[
{
"action": "click",
"target": "test.app:id/plus",
"target_description": "plus",
"times": 3,
"delay_ms": 0,
},
{"action": "click", "target": [100, 100], "target_description": "next"},
]
)
receipt = result.metadata["sequence_results"][0]
assert result.metadata["sequence_results"][1]["execution"] == "not_started"
else:
result = await actuator.click(
28, 19, times=3, delay_ms=0, expected_resource_id="test.app:id/plus"
)
receipt = result.metadata
assert not result.ok and result.code.value == "TARGET_NOT_FOUND"
assert receipt["completed_taps"] == 1
assert receipt["execution"] == "unknown"
actuator.controller.tap_at.assert_awaited_once()
@pytest.mark.asyncio
@pytest.mark.parametrize("times", [0, 11, True])
async def test_invalid_repeat_count_rejects_entire_sequence_before_input(times):
actuator = MockActuator(width=720, height=1600)
actuator.controller.tap_at = AsyncMock()
invalid = steps()
invalid[-1] = {
"action": "click",
"target": [200, 900],
"target_description": "plus",
"times": times,
}
result = await actuator.action_sequence(invalid)
assert not result.ok and result.code.value == "INVALID_ARGS"
actuator.controller.tap_at.assert_not_awaited()
@pytest.mark.parametrize("limit", [2, 4, 8])
def test_runtime_sequence_contract_matches_both_modes_and_counts_waits(limit):
from langchain_core.utils.function_calling import convert_to_openai_tool
from artemis.mcp.action_specs import operator_shell_tool, tool_declaration
from artemis.mcp.gestures import validate_sequence
pro = operator_shell_tool("action_sequence", sequence_limit=limit)
flash = tool_declaration("action_sequence", sequence_limit=limit)
assert flash.parameters == convert_to_openai_tool(pro)["function"]["parameters"]
assert flash.parameters["properties"]["steps"]["maxItems"] == limit
assert f"at most {limit} members" in pro.description
with pytest.raises(ValueError, match=f"at most {limit}, including waits"):
validate_sequence({"steps": [{"action": "wait", "time_in_ms": 1}] * 13}, limit)
@pytest.mark.asyncio
@pytest.mark.parametrize("already_open", [False, True])
async def test_guarded_popup_opener_handles_open_and_closed_without_toggling(already_open):
actuator = MockActuator(width=720, height=1600)
panel = "test.app:id/panel_control"
guarded = steps()
guarded[0]["unless_visible"] = panel
guarded[-1]["require_visible"] = panel
actuator.controller.get_ui_elements = AsyncMock(
side_effect=[
[{"resource_id": panel if already_open else "test.app:id/opener"}],
[{"resource_id": panel}],
]
)
actuator.controller.tap_at = AsyncMock(return_value=SimpleNamespace(error=None))
actuator.controller.swipe_coords = AsyncMock(return_value=None)
result = await actuator.action_sequence(guarded)
assert result.ok
assert actuator.controller.tap_at.await_count == (0 if already_open else 1)
actuator.controller.swipe_coords.assert_awaited_once()
assert actuator.controller.get_ui_elements.await_count == 2
assert result.metadata["skipped_steps"] == int(already_open)
assert result.metadata["sequence_results"][0]["execution"] == (
"skipped" if already_open else "completed"
)
@pytest.mark.asyncio
async def test_popup_expiring_before_drag_never_touches_underlying_content():
actuator = MockActuator(width=720, height=1600)
panel = "test.app:id/panel_control"
guarded = steps()
guarded[0]["unless_visible"] = panel
guarded[-1]["require_visible"] = panel
actuator.controller.get_ui_elements = AsyncMock(
side_effect=[
[{"resource_id": panel}],
[{"resource_id": "test.app:id/card"}],
]
)
actuator.controller.tap_at = AsyncMock()
actuator.controller.swipe_coords = AsyncMock()
result = await actuator.action_sequence(guarded)
assert not result.ok and result.code.value == "TARGET_NOT_FOUND"
assert result.metadata["execution"] == "not_started"
assert result.metadata["side_effect"] == "none"
assert [r["execution"] for r in result.metadata["sequence_results"]] == [
"skipped",
"completed",
"not_started",
]
actuator.controller.tap_at.assert_not_awaited()
actuator.controller.swipe_coords.assert_not_awaited()
@pytest.mark.asyncio
@pytest.mark.parametrize("hierarchy", [[], RuntimeError("hierarchy unavailable")])
async def test_unavailable_hierarchy_does_not_guess_popup_state(hierarchy):
actuator = MockActuator(width=720, height=1600)
guarded = steps()
guarded[0]["unless_visible"] = "test.app:id/panel_control"
actuator.controller.get_ui_elements = AsyncMock(
side_effect=hierarchy if isinstance(hierarchy, Exception) else lambda: hierarchy
)
actuator.controller.tap_at = AsyncMock()
actuator.controller.swipe_coords = AsyncMock()
result = await actuator.action_sequence(guarded)
assert not result.ok
assert result.metadata["execution"] == "not_started"
assert result.metadata["side_effect"] == "none"
actuator.controller.tap_at.assert_not_awaited()
actuator.controller.swipe_coords.assert_not_awaited()
@pytest.mark.asyncio
async def test_standalone_guarded_drag_round_trips_and_refuses_stale_target():
actuator = MockActuator(width=720, height=1600)
actuator.controller.get_ui_elements = AsyncMock(
return_value=[{"resource_id": "test.app:id/card"}]
)
actuator.controller.swipe_coords = AsyncMock()
session = ActionSession(server=build_action_server(actuator=actuator))
try:
await session.start()
request = {k: v for k, v in steps()[-1].items() if k != "action"}
result = await session.call(
"drag", {**request, "require_visible": "test.app:id/panel_control"}
)
assert not result.ok and result.code.value == "TARGET_NOT_FOUND"
assert result.metadata["execution"] == "not_started"
assert not result.metadata["visibility_guard"]["visible"]
actuator.controller.swipe_coords.assert_not_awaited()
finally:
await session.aclose()
@pytest.mark.asyncio
async def test_sequence_has_no_observation_between_inputs_and_returns_member_receipts():
actuator = MockActuator(width=720, height=1600)
@@ -195,7 +524,7 @@ async def test_over_limit_is_rejected_before_dispatch(monkeypatch):
)
actuator = MockActuator(width=720, height=1600)
result = await actuator.action_sequence(steps())
assert not result.ok and "limit of 2" in result.message
assert not result.ok and "at most 2, including waits" in result.message
assert actuator.action_history == []
tests/unit/test_drivers.py
+33
View File
@@ -284,6 +284,39 @@ async def test_android_driver_parses_xml_hierarchy_fallback():
assert any(element.get("text") == "Settings" for element in screen_data.ui_elements)
@pytest.mark.asyncio
async def test_android_live_targets_read_fresh_hierarchy_without_screenshot():
client = MagicMock()
client.get_hierarchy.side_effect = [
'<hierarchy><node resource-id="app:id/plus" bounds="[10,20][30,40]"/></hierarchy>',
"<hierarchy/>",
]
driver = AndroidAdbDriver("test-device", MagicMock(), ui_adb_client=client)
driver.get_screen_data = AsyncMock(side_effect=AssertionError("must not capture screen"))
present = await driver.get_ui_elements()
absent = await driver.get_ui_elements()
assert any(e.get("resource-id") == "app:id/plus" for e in present)
assert not any(e.get("resource-id") == "app:id/plus" for e in absent)
assert client.get_hierarchy.call_count == 2
client.get_screen_data.assert_not_called()
driver.get_screen_data.assert_not_called()
@pytest.mark.asyncio
async def test_android_live_hierarchy_errors_do_not_fall_back_to_stale_screen():
client = MagicMock()
client.get_hierarchy.side_effect = TimeoutError("hierarchy unavailable")
driver = AndroidAdbDriver("test-device", MagicMock(), ui_adb_client=client)
driver.get_screen_data = AsyncMock()
with pytest.raises(TimeoutError, match="hierarchy unavailable"):
await driver.get_ui_elements()
driver.get_screen_data.assert_not_called()
@pytest.mark.asyncio
async def test_find_element_prefers_fresh_bounds_over_stale_center():
"""Layout transitions must not leave dynamic element taps at old coordinates."""