fix: prepare transient controls inside native action sequences

tests/fixtures/action_surfaces/action_server_manifest.json

@@ -228,6 +228,49 @@
           "title": "DragStep",
           "type": "object"
         },
+        "EnsureVisibleStep": {
+          "additionalProperties": false,
+          "properties": {
+            "action": {
+              "const": "ensure_visible",
+              "title": "Action",
+              "type": "string"
+            },
+            "target": {
+              "description": "Panel control resource_id copied VERBATIM from observed elements, including the ':id/' separator. Never rewrite it as ':id:' or guess an ID.",
+              "minLength": 1,
+              "title": "Target",
+              "type": "string"
+            },
+            "opener": {
+              "description": "UI opener resource_id copied VERBATIM from observed elements, including ':id/' (not ':id:'). Tap ONCE only if target is absent; never use a business submission as an opener.",
+              "minLength": 1,
+              "title": "Opener",
+              "type": "string"
+            },
+            "target_description": {
+              "minLength": 1,
+              "title": "Target Description",
+              "type": "string"
+            },
+            "timeout_ms": {
+              "default": 1500,
+              "description": "Bounded live-UI wait AFTER opening, without extra model turns or repeated opener taps.",
+              "maximum": 3000,
+              "minimum": 100,
+              "title": "Timeout Ms",
+              "type": "integer"
+            }
+          },
+          "required": [
+            "action",
+            "target",
+            "opener",
+            "target_description"
+          ],
+          "title": "EnsureVisibleStep",
+          "type": "object"
+        },
         "ScaleAnchor": {
           "additionalProperties": false,
           "properties": {
@@ -284,6 +327,9 @@
               },
               {
                 "$ref": "#/$defs/WaitStep"
+              },
+              {
+                "$ref": "#/$defs/EnsureVisibleStep"
               }
             ]
           },

tests/fixtures/action_surfaces/operator_shells.json

@@ -54,11 +54,11 @@
     "type": "function",
     "function": {
       "name": "action_sequence",
-      "description": "[ACTION] Execute a short sequence of click, wait and drag steps in ONE call, in order, without model turns between members; explicit visibility guards may read the live UI locally. Use for temporary overlays: open the control, wait briefly, then operate it before it disappears. For clicks prefer target=the exact observed resource_id string: live bounds are resolved locally, refusing absent or ambiguous targets. Otherwise use observed [x,y] in the declared coordinate_space; do not confuse original screenshot pixels with 0..1000 normalized coordinates. Element indices are not accepted. Each click/drag needs target_description. All members are validated before any input. The configured fast-action limit applies. The first failure stops the remainder; receipts identify completed and unexecuted members. Never replay the whole sequence after a partial result. Every wait counts as a member. click.times (1..10) performs consecutive taps within ONE member, like the ordinary click tool; prefer this over expanding repeated +/- taps into many members/model turns. Each resource-targeted tap is revalidated locally; missing targets stop remaining taps and receipts preserve completed_taps. For maximum/minimum prefer observed native +/- controls, confirming that further same-direction adjustment stops changing handle/ticks. An estimated drag endpoint can be imprecise. For every auto-hiding adjustment, including the reverse direction, keep opener/wait/operation in one call. A prior screenshot does not prove the panel is still open. Use click.unless_visible and drag.require_visible with an observed panel resource_id to avoid toggling an open panel closed or dragging its underlying content. Never invent resource IDs. Verify actual ticks/handle endpoints before claiming maximum or minimum.",
+      "description": "[ACTION] General Android composer: execute ensure_visible, click, wait and drag steps in ONE call, in order, without model turns between members; explicit visibility guards may read the live UI locally. Use for temporary overlays: open the control, wait briefly, then operate it before it disappears. Prefer ensure_visible(target=observed panel control ID, opener=observed UI opener ID) followed by click/drag. This checks live UI, skips opening an already-visible panel, or taps its opener ONCE and waits up to timeout_ms (default 1500, max 3000) for the target. Each preparation counts as one member. Include it again in EVERY panel-operation call, even when the previous screenshot showed the panel open; do not split preparation and operation into different model turns. It never retries a business action. Copy all resource IDs VERBATIM, including ':id/' separators, never ':id:'. If preparation fails, re-observe/correct the ID; never guess fallback coordinates. Legacy click.unless_visible is ONLY for an opener, NEVER for an operation such as a +/- button: visible means SKIP, not require-visible. For clicks prefer target=the exact observed resource_id string: live bounds are resolved locally, refusing absent or ambiguous targets. Otherwise use observed [x,y] in the declared coordinate_space; do not confuse original screenshot pixels with 0..1000 normalized coordinates. Element indices are not accepted. Each click/drag needs target_description. All members are validated before any input. The configured fast-action limit applies. The first failure stops the remainder; receipts identify completed and unexecuted members. Never replay the whole sequence after a partial result. Every wait counts as a member. click.times (1..10) performs consecutive taps within ONE member, like the ordinary click tool; prefer this over expanding repeated +/- taps into many members/model turns. Each resource-targeted tap is revalidated locally; missing targets stop remaining taps and receipts preserve completed_taps. For maximum/minimum prefer observed native +/- controls, confirming that further same-direction adjustment stops changing handle/ticks. An estimated drag endpoint can be imprecise. For every auto-hiding adjustment, including the reverse direction, keep opener/wait/operation in one call. A prior screenshot does not prove the panel is still open. Use click.unless_visible and drag.require_visible with an observed panel resource_id to avoid toggling an open panel closed or dragging its underlying content. Never invent resource IDs. Verify actual ticks/handle endpoints before claiming maximum or minimum.",
       "parameters": {
         "properties": {
           "steps": {
-            "description": "Ordered click, wait and drag members.",
+            "description": "Ordered UI preparation, click, wait and drag members.",
             "items": {
               "anyOf": [
                 {
@@ -322,6 +322,49 @@
                   ],
                   "title": "WaitStep",
                   "type": "object"
+                },
+                {
+                  "additionalProperties": false,
+                  "properties": {
+                    "action": {
+                      "const": "ensure_visible",
+                      "title": "Action",
+                      "type": "string"
+                    },
+                    "target": {
+                      "description": "Panel control resource_id copied VERBATIM from observed elements, including the ':id/' separator. Never rewrite it as ':id:' or guess an ID.",
+                      "minLength": 1,
+                      "title": "Target",
+                      "type": "string"
+                    },
+                    "opener": {
+                      "description": "UI opener resource_id copied VERBATIM from observed elements, including ':id/' (not ':id:'). Tap ONCE only if target is absent; never use a business submission as an opener.",
+                      "minLength": 1,
+                      "title": "Opener",
+                      "type": "string"
+                    },
+                    "target_description": {
+                      "minLength": 1,
+                      "title": "Target Description",
+                      "type": "string"
+                    },
+                    "timeout_ms": {
+                      "default": 1500,
+                      "description": "Bounded live-UI wait AFTER opening, without extra model turns or repeated opener taps.",
+                      "maximum": 3000,
+                      "minimum": 100,
+                      "title": "Timeout Ms",
+                      "type": "integer"
+                    }
+                  },
+                  "required": [
+                    "action",
+                    "target",
+                    "opener",
+                    "target_description"
+                  ],
+                  "title": "EnsureVisibleStep",
+                  "type": "object"
                 }
               ]
             },

tests/fixtures/action_surfaces/tool_declarations.json

@@ -464,11 +464,11 @@
     "type": "function",
     "function": {
       "name": "action_sequence",
-      "description": "[ACTION] Execute a short sequence of click, wait and drag steps in ONE call, in order, without model turns between members; explicit visibility guards may read the live UI locally. Use for temporary overlays: open the control, wait briefly, then operate it before it disappears. For clicks prefer target=the exact observed resource_id string: live bounds are resolved locally, refusing absent or ambiguous targets. Otherwise use observed [x,y] in the declared coordinate_space; do not confuse original screenshot pixels with 0..1000 normalized coordinates. Element indices are not accepted. Each click/drag needs target_description. All members are validated before any input. The configured fast-action limit applies. The first failure stops the remainder; receipts identify completed and unexecuted members. Never replay the whole sequence after a partial result. Every wait counts as a member. click.times (1..10) performs consecutive taps within ONE member, like the ordinary click tool; prefer this over expanding repeated +/- taps into many members/model turns. Each resource-targeted tap is revalidated locally; missing targets stop remaining taps and receipts preserve completed_taps. For maximum/minimum prefer observed native +/- controls, confirming that further same-direction adjustment stops changing handle/ticks. An estimated drag endpoint can be imprecise. For every auto-hiding adjustment, including the reverse direction, keep opener/wait/operation in one call. A prior screenshot does not prove the panel is still open. Use click.unless_visible and drag.require_visible with an observed panel resource_id to avoid toggling an open panel closed or dragging its underlying content. Never invent resource IDs. Verify actual ticks/handle endpoints before claiming maximum or minimum.",
+      "description": "[ACTION] General Android composer: execute ensure_visible, click, wait and drag steps in ONE call, in order, without model turns between members; explicit visibility guards may read the live UI locally. Use for temporary overlays: open the control, wait briefly, then operate it before it disappears. Prefer ensure_visible(target=observed panel control ID, opener=observed UI opener ID) followed by click/drag. This checks live UI, skips opening an already-visible panel, or taps its opener ONCE and waits up to timeout_ms (default 1500, max 3000) for the target. Each preparation counts as one member. Include it again in EVERY panel-operation call, even when the previous screenshot showed the panel open; do not split preparation and operation into different model turns. It never retries a business action. Copy all resource IDs VERBATIM, including ':id/' separators, never ':id:'. If preparation fails, re-observe/correct the ID; never guess fallback coordinates. Legacy click.unless_visible is ONLY for an opener, NEVER for an operation such as a +/- button: visible means SKIP, not require-visible. For clicks prefer target=the exact observed resource_id string: live bounds are resolved locally, refusing absent or ambiguous targets. Otherwise use observed [x,y] in the declared coordinate_space; do not confuse original screenshot pixels with 0..1000 normalized coordinates. Element indices are not accepted. Each click/drag needs target_description. All members are validated before any input. The configured fast-action limit applies. The first failure stops the remainder; receipts identify completed and unexecuted members. Never replay the whole sequence after a partial result. Every wait counts as a member. click.times (1..10) performs consecutive taps within ONE member, like the ordinary click tool; prefer this over expanding repeated +/- taps into many members/model turns. Each resource-targeted tap is revalidated locally; missing targets stop remaining taps and receipts preserve completed_taps. For maximum/minimum prefer observed native +/- controls, confirming that further same-direction adjustment stops changing handle/ticks. An estimated drag endpoint can be imprecise. For every auto-hiding adjustment, including the reverse direction, keep opener/wait/operation in one call. A prior screenshot does not prove the panel is still open. Use click.unless_visible and drag.require_visible with an observed panel resource_id to avoid toggling an open panel closed or dragging its underlying content. Never invent resource IDs. Verify actual ticks/handle endpoints before claiming maximum or minimum.",
       "parameters": {
         "properties": {
           "steps": {
-            "description": "Ordered click, wait and drag members.",
+            "description": "Ordered UI preparation, click, wait and drag members.",
             "items": {
               "anyOf": [
                 {
@@ -732,6 +732,49 @@
                   ],
                   "title": "WaitStep",
                   "type": "object"
+                },
+                {
+                  "additionalProperties": false,
+                  "properties": {
+                    "action": {
+                      "const": "ensure_visible",
+                      "title": "Action",
+                      "type": "string"
+                    },
+                    "target": {
+                      "description": "Panel control resource_id copied VERBATIM from observed elements, including the ':id/' separator. Never rewrite it as ':id:' or guess an ID.",
+                      "minLength": 1,
+                      "title": "Target",
+                      "type": "string"
+                    },
+                    "opener": {
+                      "description": "UI opener resource_id copied VERBATIM from observed elements, including ':id/' (not ':id:'). Tap ONCE only if target is absent; never use a business submission as an opener.",
+                      "minLength": 1,
+                      "title": "Opener",
+                      "type": "string"
+                    },
+                    "target_description": {
+                      "minLength": 1,
+                      "title": "Target Description",
+                      "type": "string"
+                    },
+                    "timeout_ms": {
+                      "default": 1500,
+                      "description": "Bounded live-UI wait AFTER opening, without extra model turns or repeated opener taps.",
+                      "maximum": 3000,
+                      "minimum": 100,
+                      "title": "Timeout Ms",
+                      "type": "integer"
+                    }
+                  },
+                  "required": [
+                    "action",
+                    "target",
+                    "opener",
+                    "target_description"
+                  ],
+                  "title": "EnsureVisibleStep",
+                  "type": "object"
                 }
               ]
             },

tests/unit/mcp/test_action_manifest.py

@@ -64,6 +64,7 @@ NON_TOOL_IDENTIFIERS = frozenset(
         "target_description",
         "report_note_key",
         "unless_visible",
+        "ensure_visible",
         "require_visible",
         "sequence_results",
         "skipped",

tests/unit/mcp/test_gestures.py

@@ -12,6 +12,130 @@ from artemis.mcp.actuators import MockActuator
 from artemis.mcp.gestures import DragRequest, plan_drag
 
 
+def prepared_sequence():
+    return [
+        {
+            "action": "ensure_visible",
+            "target": "test.app:id/panel",
+            "opener": "test.app:id/opener",
+            "target_description": "show panel",
+            "timeout_ms": 100,
+        },
+        {
+            "action": "click",
+            "target": "test.app:id/panel",
+            "target_description": "operate panel",
+        },
+    ]
+
+
+PANEL = {"resource_id": "test.app:id/panel", "bounds": [200, 300, 300, 400]}
+OPENER = {"resource_id": "test.app:id/opener", "bounds": [100, 1400, 200, 1500]}
+
+
+@pytest.mark.asyncio
+async def test_preparation_keeps_open_panel_open_and_operates_its_live_target():
+    actuator = MockActuator(width=720, height=1600)
+    actuator.controller.get_ui_elements = AsyncMock(return_value=[OPENER, PANEL])
+    actuator.controller.tap_at = AsyncMock(return_value=SimpleNamespace(error=None))
+    result = await actuator.action_sequence(prepared_sequence())
+    assert result.ok
+    actuator.controller.tap_at.assert_awaited_once_with(250, 350)
+    assert result.metadata["sequence_results"][0]["execution"] == "skipped"
+    assert result.metadata["sequence_results"][0]["completed_taps"] == 0
+
+
+@pytest.mark.asyncio
+async def test_preparation_waits_for_delayed_panel_then_uses_current_bounds():
+    actuator = MockActuator(width=720, height=1600)
+    moved = {**PANEL, "bounds": [300, 400, 400, 500]}
+    actuator.controller.get_ui_elements = AsyncMock(
+        side_effect=[[OPENER], [OPENER], [OPENER, PANEL], [OPENER, moved]]
+    )
+    actuator.controller.tap_at = AsyncMock(return_value=SimpleNamespace(error=None))
+    sequence = prepared_sequence()
+    sequence[0]["timeout_ms"] = 500
+    result = await actuator.action_sequence(sequence)
+    assert result.ok
+    assert [c.args for c in actuator.controller.tap_at.await_args_list] == [(150, 1450), (350, 450)]
+    receipt = result.metadata["sequence_results"][0]
+    assert receipt["completed_taps"] == 1
+    assert receipt["visibility_attempts"] == 3
+    assert receipt["visibility_guard"]["visible"]
+
+
+@pytest.mark.asyncio
+async def test_preparation_timeout_never_retries_opener_or_executes_body():
+    actuator = MockActuator(width=720, height=1600)
+    actuator.controller.get_ui_elements = AsyncMock(return_value=[OPENER])
+    actuator.controller.tap_at = AsyncMock(return_value=SimpleNamespace(error=None))
+    result = await actuator.action_sequence(prepared_sequence())
+    assert not result.ok and result.code.value == "TIMEOUT"
+    actuator.controller.tap_at.assert_awaited_once_with(150, 1450)
+    assert result.metadata["side_effect"] == "possible"
+    assert [r["execution"] for r in result.metadata["sequence_results"]] == [
+        "unknown",
+        "not_started",
+    ]
+    assert result.metadata["sequence_results"][0]["completed_taps"] == 1
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("elements", [[PANEL, PANEL], [{"resource_id": "test.app:id/root"}], []])
+async def test_preparation_refuses_ambiguous_target_missing_opener_or_unavailable_ui(elements):
+    actuator = MockActuator(width=720, height=1600)
+    actuator.controller.get_ui_elements = AsyncMock(return_value=elements)
+    actuator.controller.tap_at = AsyncMock()
+    result = await actuator.action_sequence(prepared_sequence())
+    assert not result.ok and result.metadata["execution"] == "not_started"
+    assert result.metadata["side_effect"] == "none"
+    actuator.controller.tap_at.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+async def test_panel_disappearing_after_preparation_stops_without_reopening():
+    actuator = MockActuator(width=720, height=1600)
+    actuator.controller.get_ui_elements = AsyncMock(
+        side_effect=[[OPENER], [OPENER, PANEL], [OPENER]]
+    )
+    actuator.controller.tap_at = AsyncMock(return_value=SimpleNamespace(error=None))
+    result = await actuator.action_sequence(prepared_sequence())
+    assert not result.ok and result.code.value == "TARGET_NOT_FOUND"
+    actuator.controller.tap_at.assert_awaited_once_with(150, 1450)
+    assert result.metadata["sequence_results"][1]["execution"] == "not_started"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("invalid", ["same_id", "bad_late_member", "bad_timeout"])
+async def test_preparation_validates_whole_sequence_before_opening(invalid):
+    actuator = MockActuator(width=720, height=1600)
+    actuator.controller.tap_at = AsyncMock()
+    sequence = prepared_sequence()
+    if invalid == "same_id":
+        sequence[0]["opener"] = sequence[0]["target"]
+    elif invalid == "bad_timeout":
+        sequence[0]["timeout_ms"] = 3001
+    else:
+        sequence[1]["times"] = 0
+    result = await actuator.action_sequence(sequence)
+    assert not result.ok and result.code.value == "INVALID_ARGS"
+    actuator.controller.tap_at.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+async def test_preparation_round_trips_through_shared_mcp_action_server():
+    actuator = MockActuator(width=720, height=1600)
+    actuator.controller.get_ui_elements = AsyncMock(return_value=[OPENER, PANEL])
+    actuator.controller.tap_at = AsyncMock(return_value=SimpleNamespace(error=None))
+    session = await ActionSession(build_action_server(actuator)).start()
+    try:
+        result = await session.call("action_sequence", {"steps": prepared_sequence()})
+        assert result.ok
+        actuator.controller.tap_at.assert_awaited_once_with(250, 350)
+    finally:
+        await session.aclose()
+
+
 def test_pan_uses_decreasing_observed_time_scale_without_overshoot():
     request = DragRequest(
         start=[190, 1059],

tests/unit/test_llm_stream_guard.py

@@ -1,4 +1,4 @@
-"""Runaway click arguments must never reach the executor or endless retries."""
+"""Runaway native-action arguments must never reach the executor or endless retries."""
 
 from copy import deepcopy
 import json
@@ -45,25 +45,27 @@ def test_free_text_tools_and_regular_text_are_not_limited(name):
     guard.observe(delta(json.dumps({"text": text}, ensure_ascii=False), name=name))
 
 
-def test_split_and_interleaved_arguments_are_counted_per_call_without_storing_body():
+@pytest.mark.parametrize("name", ["click", "action_sequence", "drag"])
+def test_split_and_interleaved_arguments_are_counted_per_call_without_storing_body(name):
     guard = ToolArgumentGuard()
     for index in (0, 1):
-        guard.observe(delta('{"target_description":"', name="click", index=index))
+        guard.observe(delta('{"target_description":"', name=name, index=index))
     for index in (0, 1):
         guard.observe(delta("private" * 1200, index=index))
     with pytest.raises(ToolArgumentOverflow) as caught:
         guard.observe(delta("private" * 1200, index=1))
-    assert caught.value.details["tool"] == "click"
+    assert caught.value.details["tool"] == name
     assert caught.value.details["argument_chars"] > SMALL_ACTION_ARGUMENT_LIMIT
     assert "private" not in str(caught.value)
     assert "private" not in repr(guard.calls)
 
 
-def test_name_arriving_after_arguments_still_triggers_guard():
+@pytest.mark.parametrize("name", ["click", "action_sequence", "drag"])
+def test_name_arriving_after_arguments_still_triggers_guard(name):
     guard = ToolArgumentGuard()
     guard.observe(delta("x" * (SMALL_ACTION_ARGUMENT_LIMIT + 1)))
     with pytest.raises(ToolArgumentOverflow):
-        guard.observe(delta("", name="click"))
+        guard.observe(delta("", name=name))
 
 
 def test_invalid_output_retry_policy_is_bounded_and_honors_zero_retries(monkeypatch):
@@ -76,8 +78,9 @@ def test_invalid_output_retry_policy_is_bounded_and_honors_zero_retries(monkeypa
 @pytest.mark.asyncio
 @pytest.mark.parametrize("recover", [False, True])
 @pytest.mark.parametrize("first_chunk_overflows", [False, True])
+@pytest.mark.parametrize("name", ["click", "action_sequence", "drag"])
 async def test_gateway_closes_bad_stream_discards_partial_and_bounds_retry(
-    monkeypatch, recover, first_chunk_overflows
+    monkeypatch, recover, first_chunk_overflows, name
 ):
     records, resets, closed = [], [], []
 
@@ -98,13 +101,13 @@ async def test_gateway_closes_bad_stream_discards_partial_and_bounds_retry(
             self.calls += 1
             try:
                 if recover and self.calls == 2:
-                    yield delta('{"target":3}', name="click")
+                    yield delta('{"target":3}', name=name)
                     return
                 if not first_chunk_overflows:
-                    yield delta('{"target_description":"', name="click")
+                    yield delta('{"target_description":"', name=name)
                 yield delta(
                     "private" * (SMALL_ACTION_ARGUMENT_LIMIT // 7 + 1),
-                    name="click" if first_chunk_overflows else None,
+                    name=name if first_chunk_overflows else None,
                 )
                 pytest.fail("Runaway stream was consumed after the safety bound")
             finally:
@@ -117,7 +120,7 @@ async def test_gateway_closes_bad_stream_discards_partial_and_bounds_retry(
     if recover:
         response = await wrapper.complete([])
         assert response.tool_calls == [
-            {"name": "click", "args": {"target": 3}, "id": None, "type": "tool_call"}
+            {"name": name, "args": {"target": 3}, "id": None, "type": "tool_call"}
         ]
         llm._wait_for_resume.assert_not_awaited()
     else:
@@ -134,7 +137,7 @@ async def test_gateway_closes_bad_stream_discards_partial_and_bounds_retry(
         for r in records
         if r["type"] == "llm_diagnostic" and r["payload"]["phase"] == "failed"
     )
-    assert failure["stream_guard"]["tool"] == "click"
+    assert failure["stream_guard"]["tool"] == name
     assert failure["stream_guard"]["argument_chars"] > SMALL_ACTION_ARGUMENT_LIMIT
     assert failure["failure_phase"] == "generating_tool_arguments"
     assert failure["category"] == "invalid_output"