diff --git a/artemis/agents/validator/pixel_safety_net.md b/artemis/agents/validator/pixel_safety_net.md index 2156426..1fe7da9 100644 --- a/artemis/agents/validator/pixel_safety_net.md +++ b/artemis/agents/validator/pixel_safety_net.md @@ -1,11 +1,14 @@ Role -UI State Verification Expert. Compare Image 1 (Reference) and Image 2 (Current State) to check if the target clicked by the Operator at the red dot is still present and interactive. +UI State Verification Expert. The Operator decided to act at the red dot while looking at Image 1 (Reference). Image 2 (Current State) is the live screen now. Decide whether the action can still land on its intended target. + +Read the `[Target]` block first to determine which rule applies. Rules -1. Check Intent: Read the `[Planned Action & Original Thinking]` to identify what Action the Agent intended to perform and why. -2. Specific UI Elements (Priority): If targeting a specific UI control (e.g., button, icon, text, slider handle) — even if it overlays a dynamic video, map, or live stream — focus ONLY on that element's shape. Output `{"is_present": false, ...}` if that exact element has disappeared, shifted away, or is blocked. Ignore background media changes underneath. -3. Blank Space, Backgrounds & Media Surfaces: If targeting open/blank space or the body of a playing video / animation / map / live stream itself rather than a specific control drawn on top of it (e.g., tapping a full-screen video to wake its hidden controls, tapping the backdrop to dismiss a dialog), the frames are expected to differ: ignore natural video/background frame changes and the appearance or fading of overlay controls. Output `{"is_present": true, "confidence": 1.0}` unless an unexpected blocking popup now covers the tap point. +1. `Kind: specific UI control` — the target has a label, resource id, class or bounds. Judge whether that exact control remains visible at the red dot in Image 2. Ignore video, map, animation or background changes beneath it. A control that has hidden, faded, collapsed, moved or become covered is not present, even if tapping the same spot could restore it. +2. `Kind: coordinates only` — no UI hierarchy element was found at the point. Identify what appears under the red dot in both images. If Image 1 shows a distinct button, icon, link or handle, apply Rule 1. Apply the surface rule only when Image 1 shows blank space, a backdrop, or the body of a video, animation, map or live stream. +3. Surface rule. Typical intents: tap the video body to wake its hidden controls, tap the backdrop to dismiss a dialog. Surfaces are expected to change between frames: ignore natural frame changes and the appearance or fading of overlay controls. Output `{"is_present": true, "confidence": 1.0}` unless an unexpected blocking popup now covers the red dot. +4. Use `[Planned Action & Original Thinking]` to resolve whether the target was a control or its underlying surface. Do not treat a missing control as present because another tap could restore it. Output -Return EXACTLY this raw JSON and nothing else. Keep reasoning strictly under 25 words. +Return EXACTLY this raw JSON and nothing else. Keep reasoning strictly under 35 words. {"reasoning": "brief explanation", "is_present": boolean, "confidence": float} diff --git a/artemis/agents/validator/precondition_pixel.py b/artemis/agents/validator/precondition_pixel.py index 14d9806..5216a35 100644 --- a/artemis/agents/validator/precondition_pixel.py +++ b/artemis/agents/validator/precondition_pixel.py @@ -66,18 +66,47 @@ def _init_llm_and_prompt(ctx: ArtemisContext, get_llm_fn): return llm, prompt +def _describe_target(action_item: dict) -> str: + """Describe the structured action target for the pixel judge.""" + label = action_item.get("target_text") + resource_id = action_item.get("target_resource_id") + class_name = action_item.get("target_class") + bounds = action_item.get("target_bounds") + lines = ["[Target]", f"Action: {action_item.get('action')}"] + if label or resource_id or class_name or bounds: + lines.append("Kind: specific UI control") + if label: + lines.append(f"Label: {label}") + if resource_id: + lines.append(f"Resource ID: {resource_id}") + if class_name: + lines.append(f"Class: {class_name}") + if not label and not resource_id: + lines.append("Label: (unlabelled; identify it by its shape in Image 1)") + if bounds: + lines.append(f"Bounds at decision time: {bounds}") + else: + lines.append( + "Kind: coordinates only (no UI hierarchy element was found under the point;" + " inspect Image 1 for a control at the red dot)" + ) + return "\n".join(lines) + + def _build_messages( prompt: str, orig_crop_bytes: bytes, live_crop_bytes: bytes, - action_name, + action_item: dict, state: State | None, ) -> list: """Formulates the multi-modal verification message pair.""" orig_b64 = base64.b64encode(orig_crop_bytes).decode("utf-8") live_b64 = base64.b64encode(live_crop_bytes).decode("utf-8") + action_name = action_item.get("action") user_content = [ + {"type": "text", "text": _describe_target(action_item)}, {"type": "text", "text": "[Image 1 (Reference)]"}, { "type": "image_url", @@ -119,7 +148,7 @@ async def _run_attempt( prompt: str, orig_crop_bytes: bytes, current_coords, - action_name, + action_item: dict, state: State | None, attempt: int, ) -> tuple[bool, ValidationErrorCategory, str] | None: @@ -147,7 +176,7 @@ async def _run_attempt( ) # C. Formulate multi-modal verification message - messages = _build_messages(prompt, orig_crop_bytes, live_crop_bytes, action_name, state) + messages = _build_messages(prompt, orig_crop_bytes, live_crop_bytes, action_item, state) # D. Invoke Universal VLM. Parsing (fences, repair) and one # corrective re-ask on unparseable JSON are handled by the @@ -257,7 +286,7 @@ async def validate_action_precondition_pixel( prompt, orig_crop_bytes, current_coords, - action_name, + action_item, state, attempt, ) diff --git a/tests/unit/agents/test_precondition_pixel_target.py b/tests/unit/agents/test_precondition_pixel_target.py new file mode 100644 index 0000000..8006348 --- /dev/null +++ b/tests/unit/agents/test_precondition_pixel_target.py @@ -0,0 +1,100 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Tests for pixel safety-net target descriptions.""" + +from pathlib import Path +from unittest.mock import MagicMock + +from artemis.agents.validator import precondition_pixel as pp +from langchain_core.messages import HumanMessage, SystemMessage + +_PNG = b"fake-image-bytes" + + +def _texts(messages: list) -> list[str]: + human = messages[1] + return [part["text"] for part in human.content if part["type"] == "text"] + + +def test_named_control_target_block_precedes_images(): + item = { + "action": "tap", + "coordinates": [1001, 718], + "target_text": "进入全屏模式", + "target_resource_id": "com.google.android.youtube:id/fullscreen_button", + "target_class": "android.widget.ImageView", + "target_bounds": [922, 655, 1080, 781], + } + state = MagicMock() + state.operator_native_thinking = "identified 进入全屏模式" + state.operator_raw_thinking = None + + messages = pp._build_messages("SYSTEM", _PNG, _PNG, item, state) + + assert isinstance(messages[0], SystemMessage) + assert isinstance(messages[1], HumanMessage) + first_text = messages[1].content[0]["text"] + assert first_text.startswith("[Target]") + assert "Kind: specific UI control" in first_text + assert "Label: 进入全屏模式" in first_text + assert "Resource ID: com.google.android.youtube:id/fullscreen_button" in first_text + assert "Class: android.widget.ImageView" in first_text + assert "[922, 655, 1080, 781]" in first_text + assert messages[1].content[1]["text"] == "[Image 1 (Reference)]" + assert any("Original Thinking" in t for t in _texts(messages)) + assert any("Action: tap" in t for t in _texts(messages)) + + +def test_coordinates_only_target_is_labelled_as_surface(): + item = {"action": "tap", "coordinates": [540, 400]} + + messages = pp._build_messages("SYSTEM", _PNG, _PNG, item, state=None) + + first_text = messages[1].content[0]["text"] + assert "Kind: coordinates only" in first_text + assert "Label:" not in first_text + assert "Resource ID:" not in first_text + assert not any("Original Thinking" in t for t in _texts(messages)) + + +def test_label_alone_counts_as_named_control(): + item = {"action": "tap", "coordinates": [10, 10], "target_text": "Submit"} + text = pp._describe_target(item) + assert "Kind: specific UI control" in text + assert "Label: Submit" in text + assert "Resource ID" not in text + + +def test_unlabelled_hit_test_element_is_still_a_control(): + item = { + "action": "tap", + "coordinates": [1001, 718], + "target_class": "android.widget.ImageView", + "target_bounds": [922, 655, 1080, 781], + "target_label_source": "hit_test", + } + text = pp._describe_target(item) + assert "Kind: specific UI control" in text + assert "unlabelled" in text + assert "Class: android.widget.ImageView" in text + assert "[922, 655, 1080, 781]" in text + + +def test_prompt_rules_are_keyed_on_target_kind(): + prompt = Path(pp.__file__).with_name("pixel_safety_net.md").read_text(encoding="utf-8") + assert "`Kind: specific UI control`" in prompt + assert "`Kind: coordinates only`" in prompt + assert "hidden" in prompt + assert "Identify what appears under the red dot" in prompt