fix: preserve pixel safety target context

artemis/agents/validator/pixel_safety_net.md

@@ -1,11 +1,14 @@
 Role
-UI State Verification Expert. Compare Image 1 (Reference) and Image 2 (Current State) to check if the target clicked by the Operator at the red dot is still present and interactive.
+UI State Verification Expert. The Operator decided to act at the red dot while looking at Image 1 (Reference). Image 2 (Current State) is the live screen now. Decide whether the action can still land on its intended target.
+
+Read the `[Target]` block first to determine which rule applies.
 
 Rules
-1. Check Intent: Read the `[Planned Action & Original Thinking]` to identify what Action the Agent intended to perform and why.
-2. Specific UI Elements (Priority): If targeting a specific UI control (e.g., button, icon, text, slider handle) — even if it overlays a dynamic video, map, or live stream — focus ONLY on that element's shape. Output `{"is_present": false, ...}` if that exact element has disappeared, shifted away, or is blocked. Ignore background media changes underneath.
-3. Blank Space, Backgrounds & Media Surfaces: If targeting open/blank space or the body of a playing video / animation / map / live stream itself rather than a specific control drawn on top of it (e.g., tapping a full-screen video to wake its hidden controls, tapping the backdrop to dismiss a dialog), the frames are expected to differ: ignore natural video/background frame changes and the appearance or fading of overlay controls. Output `{"is_present": true, "confidence": 1.0}` unless an unexpected blocking popup now covers the tap point.
+1. `Kind: specific UI control` — the target has a label, resource id, class or bounds. Judge whether that exact control remains visible at the red dot in Image 2. Ignore video, map, animation or background changes beneath it. A control that has hidden, faded, collapsed, moved or become covered is not present, even if tapping the same spot could restore it.
+2. `Kind: coordinates only` — no UI hierarchy element was found at the point. Identify what appears under the red dot in both images. If Image 1 shows a distinct button, icon, link or handle, apply Rule 1. Apply the surface rule only when Image 1 shows blank space, a backdrop, or the body of a video, animation, map or live stream.
+3. Surface rule. Typical intents: tap the video body to wake its hidden controls, tap the backdrop to dismiss a dialog. Surfaces are expected to change between frames: ignore natural frame changes and the appearance or fading of overlay controls. Output `{"is_present": true, "confidence": 1.0}` unless an unexpected blocking popup now covers the red dot.
+4. Use `[Planned Action & Original Thinking]` to resolve whether the target was a control or its underlying surface. Do not treat a missing control as present because another tap could restore it.
 
 Output
-Return EXACTLY this raw JSON and nothing else. Keep reasoning strictly under 25 words.
+Return EXACTLY this raw JSON and nothing else. Keep reasoning strictly under 35 words.
 {"reasoning": "brief explanation", "is_present": boolean, "confidence": float}

artemis/agents/validator/precondition_pixel.py

@@ -66,18 +66,47 @@ def _init_llm_and_prompt(ctx: ArtemisContext, get_llm_fn):
     return llm, prompt
 
 
+def _describe_target(action_item: dict) -> str:
+    """Describe the structured action target for the pixel judge."""
+    label = action_item.get("target_text")
+    resource_id = action_item.get("target_resource_id")
+    class_name = action_item.get("target_class")
+    bounds = action_item.get("target_bounds")
+    lines = ["[Target]", f"Action: {action_item.get('action')}"]
+    if label or resource_id or class_name or bounds:
+        lines.append("Kind: specific UI control")
+        if label:
+            lines.append(f"Label: {label}")
+        if resource_id:
+            lines.append(f"Resource ID: {resource_id}")
+        if class_name:
+            lines.append(f"Class: {class_name}")
+        if not label and not resource_id:
+            lines.append("Label: (unlabelled; identify it by its shape in Image 1)")
+        if bounds:
+            lines.append(f"Bounds at decision time: {bounds}")
+    else:
+        lines.append(
+            "Kind: coordinates only (no UI hierarchy element was found under the point;"
+            " inspect Image 1 for a control at the red dot)"
+        )
+    return "\n".join(lines)
+
+
 def _build_messages(
     prompt: str,
     orig_crop_bytes: bytes,
     live_crop_bytes: bytes,
-    action_name,
+    action_item: dict,
     state: State | None,
 ) -> list:
     """Formulates the multi-modal verification message pair."""
     orig_b64 = base64.b64encode(orig_crop_bytes).decode("utf-8")
     live_b64 = base64.b64encode(live_crop_bytes).decode("utf-8")
+    action_name = action_item.get("action")
 
     user_content = [
+        {"type": "text", "text": _describe_target(action_item)},
         {"type": "text", "text": "[Image 1 (Reference)]"},
         {
             "type": "image_url",
@@ -119,7 +148,7 @@ async def _run_attempt(
     prompt: str,
     orig_crop_bytes: bytes,
     current_coords,
-    action_name,
+    action_item: dict,
     state: State | None,
     attempt: int,
 ) -> tuple[bool, ValidationErrorCategory, str] | None:
@@ -147,7 +176,7 @@ async def _run_attempt(
     )
 
     # C. Formulate multi-modal verification message
-    messages = _build_messages(prompt, orig_crop_bytes, live_crop_bytes, action_name, state)
+    messages = _build_messages(prompt, orig_crop_bytes, live_crop_bytes, action_item, state)
 
     # D. Invoke Universal VLM. Parsing (fences, repair) and one
     # corrective re-ask on unparseable JSON are handled by the
@@ -257,7 +286,7 @@ async def validate_action_precondition_pixel(
                 prompt,
                 orig_crop_bytes,
                 current_coords,
-                action_name,
+                action_item,
                 state,
                 attempt,
             )

tests/unit/agents/test_precondition_pixel_target.py

@@ -0,0 +1,100 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for pixel safety-net target descriptions."""
+
+from pathlib import Path
+from unittest.mock import MagicMock
+
+from artemis.agents.validator import precondition_pixel as pp
+from langchain_core.messages import HumanMessage, SystemMessage
+
+_PNG = b"fake-image-bytes"
+
+
+def _texts(messages: list) -> list[str]:
+    human = messages[1]
+    return [part["text"] for part in human.content if part["type"] == "text"]
+
+
+def test_named_control_target_block_precedes_images():
+    item = {
+        "action": "tap",
+        "coordinates": [1001, 718],
+        "target_text": "进入全屏模式",
+        "target_resource_id": "com.google.android.youtube:id/fullscreen_button",
+        "target_class": "android.widget.ImageView",
+        "target_bounds": [922, 655, 1080, 781],
+    }
+    state = MagicMock()
+    state.operator_native_thinking = "identified 进入全屏模式"
+    state.operator_raw_thinking = None
+
+    messages = pp._build_messages("SYSTEM", _PNG, _PNG, item, state)
+
+    assert isinstance(messages[0], SystemMessage)
+    assert isinstance(messages[1], HumanMessage)
+    first_text = messages[1].content[0]["text"]
+    assert first_text.startswith("[Target]")
+    assert "Kind: specific UI control" in first_text
+    assert "Label: 进入全屏模式" in first_text
+    assert "Resource ID: com.google.android.youtube:id/fullscreen_button" in first_text
+    assert "Class: android.widget.ImageView" in first_text
+    assert "[922, 655, 1080, 781]" in first_text
+    assert messages[1].content[1]["text"] == "[Image 1 (Reference)]"
+    assert any("Original Thinking" in t for t in _texts(messages))
+    assert any("Action: tap" in t for t in _texts(messages))
+
+
+def test_coordinates_only_target_is_labelled_as_surface():
+    item = {"action": "tap", "coordinates": [540, 400]}
+
+    messages = pp._build_messages("SYSTEM", _PNG, _PNG, item, state=None)
+
+    first_text = messages[1].content[0]["text"]
+    assert "Kind: coordinates only" in first_text
+    assert "Label:" not in first_text
+    assert "Resource ID:" not in first_text
+    assert not any("Original Thinking" in t for t in _texts(messages))
+
+
+def test_label_alone_counts_as_named_control():
+    item = {"action": "tap", "coordinates": [10, 10], "target_text": "Submit"}
+    text = pp._describe_target(item)
+    assert "Kind: specific UI control" in text
+    assert "Label: Submit" in text
+    assert "Resource ID" not in text
+
+
+def test_unlabelled_hit_test_element_is_still_a_control():
+    item = {
+        "action": "tap",
+        "coordinates": [1001, 718],
+        "target_class": "android.widget.ImageView",
+        "target_bounds": [922, 655, 1080, 781],
+        "target_label_source": "hit_test",
+    }
+    text = pp._describe_target(item)
+    assert "Kind: specific UI control" in text
+    assert "unlabelled" in text
+    assert "Class: android.widget.ImageView" in text
+    assert "[922, 655, 1080, 781]" in text
+
+
+def test_prompt_rules_are_keyed_on_target_kind():
+    prompt = Path(pp.__file__).with_name("pixel_safety_net.md").read_text(encoding="utf-8")
+    assert "`Kind: specific UI control`" in prompt
+    assert "`Kind: coordinates only`" in prompt
+    assert "hidden" in prompt
+    assert "Identify what appears under the red dot" in prompt