fix: guard transient controls and verify timeline endpoints

tests/unit/agents/test_flash_runner.py

@@ -134,7 +134,9 @@ async def test_fallback_observation_renders_the_indexed_ui_list(mock_context, tm
 
     assert post == _POST_IMG
     assert isinstance(exec_result.ui_elements_text, str)
-    assert exec_result.ui_elements_text == "[1] Text: 'Settings' | Bounds: [37,50][962,112]"
+    assert exec_result.ui_elements_text == (
+        "[1] Text: 'Settings' | Bounds: [37,50][962,112] | resource_id: 'android:id/title'"
+    )
     assert "{'text'" not in exec_result.ui_elements_text
     # State refreshed exactly as the executor's success path does.
     assert state.indexed_elements[0]["index"] == 1

tests/unit/agents/test_validator.py

@@ -23,6 +23,7 @@ feeds the safety-net XML validation.
 
 import json
 from unittest.mock import AsyncMock, MagicMock, Mock, patch
+from uuid import uuid4
 
 from langchain_core.messages import AIMessage
 import pytest
@@ -309,6 +310,48 @@ async def test_validator_burst_success_executes_every_member(
     assert len(mock_mcp.calls_for("click")) == 2
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("post_image", ["screenshot", "new-sequence-image"])
+async def test_sequence_captures_once_after_whole_burst_and_retains_equal_hash(
+    mock_mcp, mock_context, temp_screenshot, post_image
+):
+    events = []
+
+    def action(name, args):
+        events.append(name)
+        return True, ""
+
+    def screenshot():
+        events.append("capture")
+        return "ZHVtbXlfZGF0YQ=="
+
+    mock_mcp.action_handler = action
+    mock_mcp.screenshot = screenshot
+    mock_context.data_engine.get_or_create_image.return_value = post_image
+    mock_context.data_engine.get_image_path.return_value = temp_screenshot
+    state = DummyState(
+        structured_decisions=json.dumps(
+            [
+                {
+                    "action": "action_sequence",
+                    "args": {"steps": [{"action": "wait", "time_in_ms": 1}]},
+                },
+                {"action": "tap", "coordinates": [105, 205]},
+            ]
+        ),
+        latest_screenshot=temp_screenshot,
+        current_step_id=str(uuid4()),
+    )
+
+    result = await ValidatorNode(mock_context)(state)
+
+    assert result["last_execution_result"]["status"] == "dispatched"
+    assert events == ["action_sequence", "click", "capture"]
+    assert mock_context.data_engine.update_step_execution_result.call_args.kwargs == {
+        "post_image_name": post_image
+    }
+
+
 @pytest.mark.asyncio
 async def test_validator_success_closes_open_incident(mock_mcp, mock_context, temp_screenshot):
     """A successful turn closes whatever incident was still open."""

tests/unit/mcp/test_action_executor_semantics.py

@@ -111,7 +111,12 @@ def test_click_index_resolves_to_the_elements_center_and_records_observed_fields
     )
     assert wire_name == "click"
     # 540/1080 -> 500, 1248/2400 -> 520 (normalized 0-1000)
-    assert wire_args == {"target": [500, 520], "times": 1, "delay_ms": 100}
+    assert wire_args == {
+        "target": [500, 520],
+        "times": 1,
+        "delay_ms": 100,
+        "expected_resource_id": "android:id/title",
+    }
     assert recorded == {
         "target_text": "Wi-Fi",
         "target_bounds": [37, 480, 963, 560],

tests/unit/mcp/test_action_manifest.py

@@ -57,7 +57,18 @@ SCOPED_PROMPT_FILES = [
 # decision.
 #: Backticked identifiers in prompts that are parameters or note keys, not tools.
 NON_TOOL_IDENTIFIERS = frozenset(
-    {"analysis", "task_plan", "tool_enum", "target_description", "report_note_key"}
+    {
+        "analysis",
+        "task_plan",
+        "tool_enum",
+        "target_description",
+        "report_note_key",
+        "unless_visible",
+        "require_visible",
+        "sequence_results",
+        "skipped",
+        "not_started",
+    }
 )
 
 _TOOL_REF = re.compile(r"(?:`([a-z][a-z0-9_]{2,})`|\b([a-z][a-z0-9_]{2,})\()")

tests/unit/mcp/test_action_observation_settling.py

@@ -90,6 +90,20 @@ async def test_old_bitmap_with_new_hierarchy_is_never_marked_settled(tmp_path):
     assert session.observe.await_count == 4
 
 
+@pytest.mark.asyncio
+async def test_sequence_preserves_immediate_popup_evidence_without_claiming_stability(tmp_path):
+    popup = _observation(_screen(tmp_path, "popup.jpg"), "缩放面板")
+    session = Mock()
+    session.observe = AsyncMock(return_value=popup)
+    executor = McpActionExecutor(Mock(), actuator=Mock())
+
+    observed, settled = await executor._observe_after_action(session, "action_sequence", None)
+
+    assert observed is popup
+    assert settled is None
+    session.observe.assert_awaited_once_with(settle_ms=0)
+
+
 @pytest.mark.asyncio
 async def test_read_only_action_keeps_single_observation(tmp_path):
     home = _screen(tmp_path, "home.jpg")

tests/unit/mcp/test_gestures.py

@@ -101,6 +101,335 @@ def steps():
     ]
 
 
+def test_both_observation_lists_expose_real_resource_ids_without_changing_indices():
+    from artemis.utils.visualization import (
+        format_minimal_list_with_elements,
+        format_minimal_list_with_points,
+    )
+
+    nodes = [
+        {"text": "plus", "resource-id": "test.app:id/plus", "bounds": "[10,20][30,40]"},
+        {"text": "minus", "resource_id": "test.app:id/minus", "bounds": "[10,60][30,80]"},
+    ]
+    for formatter in (format_minimal_list_with_points, format_minimal_list_with_elements):
+        text, _, labels = formatter(nodes, 100, 100)
+        assert (
+            "[1] Text: 'plus' | Bounds: [100,200][300,400] | resource_id: 'test.app:id/plus'"
+            in text
+        )
+        assert "[2] Text: 'minus'" in text and "resource_id: 'test.app:id/minus'" in text
+        assert labels == ["1", "2"]
+
+
+@pytest.mark.asyncio
+async def test_live_resource_click_uses_current_bounds_and_reuses_guard_observation():
+    actuator = MockActuator(width=720, height=1600)
+    panel, opener = "test.app:id/panel", "test.app:id/opener"
+    actuator.controller.get_ui_elements = AsyncMock(
+        return_value=[
+            {"resource_id": opener, "bounds": [100, 1400, 200, 1500]},
+        ]
+    )
+    actuator.controller.tap_at = AsyncMock(return_value=SimpleNamespace(error=None))
+    result = await actuator.action_sequence(
+        [
+            {
+                "action": "click",
+                "target": opener,
+                "target_description": "open",
+                "unless_visible": panel,
+            }
+        ]
+    )
+    assert result.ok
+    actuator.controller.tap_at.assert_awaited_once_with(150, 1450)
+    actuator.controller.get_ui_elements.assert_awaited_once()
+    receipt = result.metadata["sequence_results"][0]
+    assert receipt["coordinates"] == [150, 1450]
+    assert receipt["target_resource_id"] == opener
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("matches", [0, 2])
+async def test_missing_or_ambiguous_resource_click_never_guesses_coordinates(matches):
+    actuator = MockActuator(width=720, height=1600)
+    target = "test.app:id/control"
+    actuator.controller.get_ui_elements = AsyncMock(
+        return_value=[
+            {
+                "resource_id": target if i < matches else "test.app:id/other",
+                "bounds": [10, 20, 30, 40],
+            }
+            for i in range(2)
+        ]
+    )
+    actuator.controller.tap_at = AsyncMock()
+    result = await actuator.action_sequence(
+        [{"action": "click", "target": target, "target_description": "control"}]
+    )
+    assert not result.ok and result.code.value == "TARGET_NOT_FOUND"
+    assert result.metadata["execution"] == "not_started"
+    actuator.controller.tap_at.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "resource_id,bounds,success",
+    [
+        ("test.app:id/control", [100, 1400, 200, 1500], True),
+        ("test.app:id/other", [100, 1400, 200, 1500], False),
+        ("test.app:id/control", [300, 1400, 400, 1500], False),
+    ],
+)
+async def test_indexed_click_checks_live_identity_at_original_position(
+    resource_id, bounds, success
+):
+    actuator = MockActuator(width=720, height=1600)
+    actuator.controller.get_ui_elements = AsyncMock(
+        return_value=[{"resource_id": resource_id, "bounds": bounds}]
+    )
+    actuator.controller.tap_at = AsyncMock(return_value=SimpleNamespace(error=None))
+    result = await actuator.click(217, 906, expected_resource_id="test.app:id/control")
+    assert result.ok is success
+    assert actuator.controller.tap_at.await_count == int(success)
+    if success:
+        from artemis.agents.flash.runner import FlashRunner
+
+        assert result.metadata["pixel_coordinates"] == [156, 1449]
+        assert "coordinates" not in result.metadata
+        runner = object.__new__(FlashRunner)
+        record = runner._build_action_record(
+            "click", {"target": [217, 906]}, SimpleNamespace(metadata=result.metadata)
+        )
+        assert record["coordinates"] == [217, 906]
+        assert record["coordinate_space"] == "normalized"
+        assert record["completed_taps"] == 1
+        assert record["pixel_coordinates"] == [156, 1449]
+    else:
+        assert result.metadata["execution"] == "not_started"
+        assert result.metadata["side_effect"] == "none"
+
+
+@pytest.mark.asyncio
+async def test_coordinate_only_click_keeps_existing_direct_dispatch():
+    actuator = MockActuator(width=720, height=1600)
+    actuator.controller.get_ui_elements = AsyncMock()
+    result = await actuator.click(217, 906)
+    assert result.ok
+    actuator.controller.get_ui_elements.assert_not_awaited()
+
+
+def test_flash_and_pro_forward_observed_resource_identity_without_inventing_it():
+    from artemis.mcp.action_executor import McpActionExecutor
+    from artemis.mcp.action_names import to_canonical_call
+
+    executor = object.__new__(McpActionExecutor)
+    executor._resolve_target = MagicMock(
+        return_value=([217, 906], {"target_resource_id": "test.app:id/control"})
+    )
+    name, args, _, _ = executor._translate("click", {"target": 1}, MagicMock())
+    assert name == "click" and args["expected_resource_id"] == "test.app:id/control"
+    _, args = to_canonical_call(
+        {
+            "action": "tap",
+            "normalized_coordinates": [217, 906],
+            "resource_id": "test.app:id/control",
+        }
+    )
+    assert args["expected_resource_id"] == "test.app:id/control"
+
+
+@pytest.mark.asyncio
+async def test_repeated_resource_taps_are_one_bounded_sequence_member():
+    actuator = MockActuator(width=720, height=1600)
+    actuator.controller.get_ui_elements = AsyncMock(
+        return_value=[{"resource_id": "test.app:id/plus", "bounds": [10, 20, 30, 40]}]
+    )
+    actuator.controller.tap_at = AsyncMock(return_value=SimpleNamespace(error=None))
+    result = await actuator.action_sequence(
+        [
+            {
+                "action": "click",
+                "target": "test.app:id/plus",
+                "target_description": "plus",
+                "times": 10,
+                "delay_ms": 50,
+            }
+        ]
+    )
+    assert result.ok and result.metadata["completed_steps"] == 1
+    assert actuator.controller.tap_at.await_count == 10
+    actuator.controller.tap_at.assert_awaited_with(20, 30)
+    assert actuator.controller.get_ui_elements.await_count == 10
+    receipt = result.metadata["sequence_results"][0]
+    assert receipt["times"] == receipt["completed_taps"] == 10
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("sequence", [True, False])
+async def test_repeated_native_taps_stop_when_target_disappears(sequence):
+    actuator = MockActuator(width=720, height=1600)
+    actuator.controller.get_ui_elements = AsyncMock(
+        side_effect=[
+            [{"resource_id": "test.app:id/plus", "bounds": [10, 20, 30, 40]}],
+            [{"resource_id": "test.app:id/background", "bounds": [0, 0, 720, 1600]}],
+        ]
+    )
+    actuator.controller.tap_at = AsyncMock(return_value=SimpleNamespace(error=None))
+    if sequence:
+        result = await actuator.action_sequence(
+            [
+                {
+                    "action": "click",
+                    "target": "test.app:id/plus",
+                    "target_description": "plus",
+                    "times": 3,
+                    "delay_ms": 0,
+                },
+                {"action": "click", "target": [100, 100], "target_description": "next"},
+            ]
+        )
+        receipt = result.metadata["sequence_results"][0]
+        assert result.metadata["sequence_results"][1]["execution"] == "not_started"
+    else:
+        result = await actuator.click(
+            28, 19, times=3, delay_ms=0, expected_resource_id="test.app:id/plus"
+        )
+        receipt = result.metadata
+    assert not result.ok and result.code.value == "TARGET_NOT_FOUND"
+    assert receipt["completed_taps"] == 1
+    assert receipt["execution"] == "unknown"
+    actuator.controller.tap_at.assert_awaited_once()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("times", [0, 11, True])
+async def test_invalid_repeat_count_rejects_entire_sequence_before_input(times):
+    actuator = MockActuator(width=720, height=1600)
+    actuator.controller.tap_at = AsyncMock()
+    invalid = steps()
+    invalid[-1] = {
+        "action": "click",
+        "target": [200, 900],
+        "target_description": "plus",
+        "times": times,
+    }
+    result = await actuator.action_sequence(invalid)
+    assert not result.ok and result.code.value == "INVALID_ARGS"
+    actuator.controller.tap_at.assert_not_awaited()
+
+
+@pytest.mark.parametrize("limit", [2, 4, 8])
+def test_runtime_sequence_contract_matches_both_modes_and_counts_waits(limit):
+    from langchain_core.utils.function_calling import convert_to_openai_tool
+    from artemis.mcp.action_specs import operator_shell_tool, tool_declaration
+    from artemis.mcp.gestures import validate_sequence
+
+    pro = operator_shell_tool("action_sequence", sequence_limit=limit)
+    flash = tool_declaration("action_sequence", sequence_limit=limit)
+    assert flash.parameters == convert_to_openai_tool(pro)["function"]["parameters"]
+    assert flash.parameters["properties"]["steps"]["maxItems"] == limit
+    assert f"at most {limit} members" in pro.description
+    with pytest.raises(ValueError, match=f"at most {limit}, including waits"):
+        validate_sequence({"steps": [{"action": "wait", "time_in_ms": 1}] * 13}, limit)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("already_open", [False, True])
+async def test_guarded_popup_opener_handles_open_and_closed_without_toggling(already_open):
+    actuator = MockActuator(width=720, height=1600)
+    panel = "test.app:id/panel_control"
+    guarded = steps()
+    guarded[0]["unless_visible"] = panel
+    guarded[-1]["require_visible"] = panel
+    actuator.controller.get_ui_elements = AsyncMock(
+        side_effect=[
+            [{"resource_id": panel if already_open else "test.app:id/opener"}],
+            [{"resource_id": panel}],
+        ]
+    )
+    actuator.controller.tap_at = AsyncMock(return_value=SimpleNamespace(error=None))
+    actuator.controller.swipe_coords = AsyncMock(return_value=None)
+    result = await actuator.action_sequence(guarded)
+    assert result.ok
+    assert actuator.controller.tap_at.await_count == (0 if already_open else 1)
+    actuator.controller.swipe_coords.assert_awaited_once()
+    assert actuator.controller.get_ui_elements.await_count == 2
+    assert result.metadata["skipped_steps"] == int(already_open)
+    assert result.metadata["sequence_results"][0]["execution"] == (
+        "skipped" if already_open else "completed"
+    )
+
+
+@pytest.mark.asyncio
+async def test_popup_expiring_before_drag_never_touches_underlying_content():
+    actuator = MockActuator(width=720, height=1600)
+    panel = "test.app:id/panel_control"
+    guarded = steps()
+    guarded[0]["unless_visible"] = panel
+    guarded[-1]["require_visible"] = panel
+    actuator.controller.get_ui_elements = AsyncMock(
+        side_effect=[
+            [{"resource_id": panel}],
+            [{"resource_id": "test.app:id/card"}],
+        ]
+    )
+    actuator.controller.tap_at = AsyncMock()
+    actuator.controller.swipe_coords = AsyncMock()
+    result = await actuator.action_sequence(guarded)
+    assert not result.ok and result.code.value == "TARGET_NOT_FOUND"
+    assert result.metadata["execution"] == "not_started"
+    assert result.metadata["side_effect"] == "none"
+    assert [r["execution"] for r in result.metadata["sequence_results"]] == [
+        "skipped",
+        "completed",
+        "not_started",
+    ]
+    actuator.controller.tap_at.assert_not_awaited()
+    actuator.controller.swipe_coords.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("hierarchy", [[], RuntimeError("hierarchy unavailable")])
+async def test_unavailable_hierarchy_does_not_guess_popup_state(hierarchy):
+    actuator = MockActuator(width=720, height=1600)
+    guarded = steps()
+    guarded[0]["unless_visible"] = "test.app:id/panel_control"
+    actuator.controller.get_ui_elements = AsyncMock(
+        side_effect=hierarchy if isinstance(hierarchy, Exception) else lambda: hierarchy
+    )
+    actuator.controller.tap_at = AsyncMock()
+    actuator.controller.swipe_coords = AsyncMock()
+    result = await actuator.action_sequence(guarded)
+    assert not result.ok
+    assert result.metadata["execution"] == "not_started"
+    assert result.metadata["side_effect"] == "none"
+    actuator.controller.tap_at.assert_not_awaited()
+    actuator.controller.swipe_coords.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+async def test_standalone_guarded_drag_round_trips_and_refuses_stale_target():
+    actuator = MockActuator(width=720, height=1600)
+    actuator.controller.get_ui_elements = AsyncMock(
+        return_value=[{"resource_id": "test.app:id/card"}]
+    )
+    actuator.controller.swipe_coords = AsyncMock()
+    session = ActionSession(server=build_action_server(actuator=actuator))
+    try:
+        await session.start()
+        request = {k: v for k, v in steps()[-1].items() if k != "action"}
+        result = await session.call(
+            "drag", {**request, "require_visible": "test.app:id/panel_control"}
+        )
+        assert not result.ok and result.code.value == "TARGET_NOT_FOUND"
+        assert result.metadata["execution"] == "not_started"
+        assert not result.metadata["visibility_guard"]["visible"]
+        actuator.controller.swipe_coords.assert_not_awaited()
+    finally:
+        await session.aclose()
+
+
 @pytest.mark.asyncio
 async def test_sequence_has_no_observation_between_inputs_and_returns_member_receipts():
     actuator = MockActuator(width=720, height=1600)
@@ -195,7 +524,7 @@ async def test_over_limit_is_rejected_before_dispatch(monkeypatch):
     )
     actuator = MockActuator(width=720, height=1600)
     result = await actuator.action_sequence(steps())
-    assert not result.ok and "limit of 2" in result.message
+    assert not result.ok and "at most 2, including waits" in result.message
     assert actuator.action_history == []
 
 

tests/unit/test_drivers.py

@@ -284,6 +284,39 @@ async def test_android_driver_parses_xml_hierarchy_fallback():
     assert any(element.get("text") == "Settings" for element in screen_data.ui_elements)
 
 
+@pytest.mark.asyncio
+async def test_android_live_targets_read_fresh_hierarchy_without_screenshot():
+    client = MagicMock()
+    client.get_hierarchy.side_effect = [
+        '<hierarchy><node resource-id="app:id/plus" bounds="[10,20][30,40]"/></hierarchy>',
+        "<hierarchy/>",
+    ]
+    driver = AndroidAdbDriver("test-device", MagicMock(), ui_adb_client=client)
+    driver.get_screen_data = AsyncMock(side_effect=AssertionError("must not capture screen"))
+
+    present = await driver.get_ui_elements()
+    absent = await driver.get_ui_elements()
+
+    assert any(e.get("resource-id") == "app:id/plus" for e in present)
+    assert not any(e.get("resource-id") == "app:id/plus" for e in absent)
+    assert client.get_hierarchy.call_count == 2
+    client.get_screen_data.assert_not_called()
+    driver.get_screen_data.assert_not_called()
+
+
+@pytest.mark.asyncio
+async def test_android_live_hierarchy_errors_do_not_fall_back_to_stale_screen():
+    client = MagicMock()
+    client.get_hierarchy.side_effect = TimeoutError("hierarchy unavailable")
+    driver = AndroidAdbDriver("test-device", MagicMock(), ui_adb_client=client)
+    driver.get_screen_data = AsyncMock()
+
+    with pytest.raises(TimeoutError, match="hierarchy unavailable"):
+        await driver.get_ui_elements()
+
+    driver.get_screen_data.assert_not_called()
+
+
 @pytest.mark.asyncio
 async def test_find_element_prefers_fresh_bounds_over_stale_center():
     """Layout transitions must not leave dynamic element taps at old coordinates."""